Index: third_party/sqlite/amalgamation/sqlite3.09.c |
diff --git a/third_party/sqlite/amalgamation/sqlite3.09.c b/third_party/sqlite/amalgamation/sqlite3.09.c |
new file mode 100644 |
index 0000000000000000000000000000000000000000..0bd404c9eadc1780bd8fa53bc2b1c3a37c22f435 |
--- /dev/null |
+++ b/third_party/sqlite/amalgamation/sqlite3.09.c |
@@ -0,0 +1,20938 @@ |
+/************** Begin file fts5.c ********************************************/ |
+ |
+ |
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) |
+ |
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) |
+# define NDEBUG 1 |
+#endif |
+#if defined(NDEBUG) && defined(SQLITE_DEBUG) |
+# undef NDEBUG |
+#endif |
+ |
+/* |
+** 2014 May 31 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** Interfaces to extend FTS5. Using the interfaces defined in this file, |
+** FTS5 may be extended with: |
+** |
+** * custom tokenizers, and |
+** * custom auxiliary functions. |
+*/ |
+ |
+ |
+#ifndef _FTS5_H |
+#define _FTS5_H |
+ |
+/* #include "sqlite3.h" */ |
+ |
+#if 0 |
+extern "C" { |
+#endif |
+ |
+/************************************************************************* |
+** CUSTOM AUXILIARY FUNCTIONS |
+** |
+** Virtual table implementations may overload SQL functions by implementing |
+** the sqlite3_module.xFindFunction() method. |
+*/ |
+ |
+typedef struct Fts5ExtensionApi Fts5ExtensionApi; |
+typedef struct Fts5Context Fts5Context; |
+typedef struct Fts5PhraseIter Fts5PhraseIter; |
+ |
+typedef void (*fts5_extension_function)( |
+ const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ |
+ Fts5Context *pFts, /* First arg to pass to pApi functions */ |
+ sqlite3_context *pCtx, /* Context for returning result/error */ |
+ int nVal, /* Number of values in apVal[] array */ |
+ sqlite3_value **apVal /* Array of trailing arguments */ |
+); |
+ |
+struct Fts5PhraseIter { |
+ const unsigned char *a; |
+ const unsigned char *b; |
+}; |
+ |
+/* |
+** EXTENSION API FUNCTIONS |
+** |
+** xUserData(pFts): |
+** Return a copy of the context pointer the extension function was |
+** registered with. |
+** |
+** xColumnTotalSize(pFts, iCol, pnToken): |
+** If parameter iCol is less than zero, set output variable *pnToken |
+** to the total number of tokens in the FTS5 table. Or, if iCol is |
+** non-negative but less than the number of columns in the table, return |
+** the total number of tokens in column iCol, considering all rows in |
+** the FTS5 table. |
+** |
+** If parameter iCol is greater than or equal to the number of columns |
+** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. |
+** an OOM condition or IO error), an appropriate SQLite error code is |
+** returned. |
+** |
+** xColumnCount(pFts): |
+** Return the number of columns in the table. |
+** |
+** xColumnSize(pFts, iCol, pnToken): |
+** If parameter iCol is less than zero, set output variable *pnToken |
+** to the total number of tokens in the current row. Or, if iCol is |
+** non-negative but less than the number of columns in the table, set |
+** *pnToken to the number of tokens in column iCol of the current row. |
+** |
+** If parameter iCol is greater than or equal to the number of columns |
+** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. |
+** an OOM condition or IO error), an appropriate SQLite error code is |
+** returned. |
+** |
+** This function may be quite inefficient if used with an FTS5 table |
+** created with the "columnsize=0" option. |
+** |
+** xColumnText: |
+** This function attempts to retrieve the text of column iCol of the |
+** current document. If successful, (*pz) is set to point to a buffer |
+** containing the text in utf-8 encoding, (*pn) is set to the size in bytes |
+** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, |
+** if an error occurs, an SQLite error code is returned and the final values |
+** of (*pz) and (*pn) are undefined. |
+** |
+** xPhraseCount: |
+** Returns the number of phrases in the current query expression. |
+** |
+** xPhraseSize: |
+** Returns the number of tokens in phrase iPhrase of the query. Phrases |
+** are numbered starting from zero. |
+** |
+** xInstCount: |
+** Set *pnInst to the total number of occurrences of all phrases within |
+** the query within the current row. Return SQLITE_OK if successful, or |
+** an error code (i.e. SQLITE_NOMEM) if an error occurs. |
+** |
+** This API can be quite slow if used with an FTS5 table created with the |
+** "detail=none" or "detail=column" option. If the FTS5 table is created |
+** with either "detail=none" or "detail=column" and "content=" option |
+** (i.e. if it is a contentless table), then this API always returns 0. |
+** |
+** xInst: |
+** Query for the details of phrase match iIdx within the current row. |
+** Phrase matches are numbered starting from zero, so the iIdx argument |
+** should be greater than or equal to zero and smaller than the value |
+** output by xInstCount(). |
+** |
+** Usually, output parameter *piPhrase is set to the phrase number, *piCol |
+** to the column in which it occurs and *piOff the token offset of the |
+** first token of the phrase. The exception is if the table was created |
+** with the offsets=0 option specified. In this case *piOff is always |
+** set to -1. |
+** |
+** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) |
+** if an error occurs. |
+** |
+** This API can be quite slow if used with an FTS5 table created with the |
+** "detail=none" or "detail=column" option. |
+** |
+** xRowid: |
+** Returns the rowid of the current row. |
+** |
+** xTokenize: |
+** Tokenize text using the tokenizer belonging to the FTS5 table. |
+** |
+** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): |
+** This API function is used to query the FTS table for phrase iPhrase |
+** of the current query. Specifically, a query equivalent to: |
+** |
+** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid |
+** |
+** with $p set to a phrase equivalent to the phrase iPhrase of the |
+** current query is executed. Any column filter that applies to |
+** phrase iPhrase of the current query is included in $p. For each |
+** row visited, the callback function passed as the fourth argument |
+** is invoked. The context and API objects passed to the callback |
+** function may be used to access the properties of each matched row. |
+** Invoking Api.xUserData() returns a copy of the pointer passed as |
+** the third argument to pUserData. |
+** |
+** If the callback function returns any value other than SQLITE_OK, the |
+** query is abandoned and the xQueryPhrase function returns immediately. |
+** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. |
+** Otherwise, the error code is propagated upwards. |
+** |
+** If the query runs to completion without incident, SQLITE_OK is returned. |
+** Or, if some error occurs before the query completes or is aborted by |
+** the callback, an SQLite error code is returned. |
+** |
+** |
+** xSetAuxdata(pFts5, pAux, xDelete) |
+** |
+** Save the pointer passed as the second argument as the extension functions |
+** "auxiliary data". The pointer may then be retrieved by the current or any |
+** future invocation of the same fts5 extension function made as part of |
+** of the same MATCH query using the xGetAuxdata() API. |
+** |
+** Each extension function is allocated a single auxiliary data slot for |
+** each FTS query (MATCH expression). If the extension function is invoked |
+** more than once for a single FTS query, then all invocations share a |
+** single auxiliary data context. |
+** |
+** If there is already an auxiliary data pointer when this function is |
+** invoked, then it is replaced by the new pointer. If an xDelete callback |
+** was specified along with the original pointer, it is invoked at this |
+** point. |
+** |
+** The xDelete callback, if one is specified, is also invoked on the |
+** auxiliary data pointer after the FTS5 query has finished. |
+** |
+** If an error (e.g. an OOM condition) occurs within this function, an |
+** the auxiliary data is set to NULL and an error code returned. If the |
+** xDelete parameter was not NULL, it is invoked on the auxiliary data |
+** pointer before returning. |
+** |
+** |
+** xGetAuxdata(pFts5, bClear) |
+** |
+** Returns the current auxiliary data pointer for the fts5 extension |
+** function. See the xSetAuxdata() method for details. |
+** |
+** If the bClear argument is non-zero, then the auxiliary data is cleared |
+** (set to NULL) before this function returns. In this case the xDelete, |
+** if any, is not invoked. |
+** |
+** |
+** xRowCount(pFts5, pnRow) |
+** |
+** This function is used to retrieve the total number of rows in the table. |
+** In other words, the same value that would be returned by: |
+** |
+** SELECT count(*) FROM ftstable; |
+** |
+** xPhraseFirst() |
+** This function is used, along with type Fts5PhraseIter and the xPhraseNext |
+** method, to iterate through all instances of a single query phrase within |
+** the current row. This is the same information as is accessible via the |
+** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient |
+** to use, this API may be faster under some circumstances. To iterate |
+** through instances of phrase iPhrase, use the following code: |
+** |
+** Fts5PhraseIter iter; |
+** int iCol, iOff; |
+** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); |
+** iCol>=0; |
+** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) |
+** ){ |
+** // An instance of phrase iPhrase at offset iOff of column iCol |
+** } |
+** |
+** The Fts5PhraseIter structure is defined above. Applications should not |
+** modify this structure directly - it should only be used as shown above |
+** with the xPhraseFirst() and xPhraseNext() API methods (and by |
+** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). |
+** |
+** This API can be quite slow if used with an FTS5 table created with the |
+** "detail=none" or "detail=column" option. If the FTS5 table is created |
+** with either "detail=none" or "detail=column" and "content=" option |
+** (i.e. if it is a contentless table), then this API always iterates |
+** through an empty set (all calls to xPhraseFirst() set iCol to -1). |
+** |
+** xPhraseNext() |
+** See xPhraseFirst above. |
+** |
+** xPhraseFirstColumn() |
+** This function and xPhraseNextColumn() are similar to the xPhraseFirst() |
+** and xPhraseNext() APIs described above. The difference is that instead |
+** of iterating through all instances of a phrase in the current row, these |
+** APIs are used to iterate through the set of columns in the current row |
+** that contain one or more instances of a specified phrase. For example: |
+** |
+** Fts5PhraseIter iter; |
+** int iCol; |
+** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); |
+** iCol>=0; |
+** pApi->xPhraseNextColumn(pFts, &iter, &iCol) |
+** ){ |
+** // Column iCol contains at least one instance of phrase iPhrase |
+** } |
+** |
+** This API can be quite slow if used with an FTS5 table created with the |
+** "detail=none" option. If the FTS5 table is created with either |
+** "detail=none" "content=" option (i.e. if it is a contentless table), |
+** then this API always iterates through an empty set (all calls to |
+** xPhraseFirstColumn() set iCol to -1). |
+** |
+** The information accessed using this API and its companion |
+** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext |
+** (or xInst/xInstCount). The chief advantage of this API is that it is |
+** significantly more efficient than those alternatives when used with |
+** "detail=column" tables. |
+** |
+** xPhraseNextColumn() |
+** See xPhraseFirstColumn above. |
+*/ |
+struct Fts5ExtensionApi { |
+ int iVersion; /* Currently always set to 3 */ |
+ |
+ void *(*xUserData)(Fts5Context*); |
+ |
+ int (*xColumnCount)(Fts5Context*); |
+ int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); |
+ int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); |
+ |
+ int (*xTokenize)(Fts5Context*, |
+ const char *pText, int nText, /* Text to tokenize */ |
+ void *pCtx, /* Context passed to xToken() */ |
+ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ |
+ ); |
+ |
+ int (*xPhraseCount)(Fts5Context*); |
+ int (*xPhraseSize)(Fts5Context*, int iPhrase); |
+ |
+ int (*xInstCount)(Fts5Context*, int *pnInst); |
+ int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); |
+ |
+ sqlite3_int64 (*xRowid)(Fts5Context*); |
+ int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); |
+ int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); |
+ |
+ int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, |
+ int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) |
+ ); |
+ int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); |
+ void *(*xGetAuxdata)(Fts5Context*, int bClear); |
+ |
+ int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); |
+ void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); |
+ |
+ int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); |
+ void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); |
+}; |
+ |
+/* |
+** CUSTOM AUXILIARY FUNCTIONS |
+*************************************************************************/ |
+ |
+/************************************************************************* |
+** CUSTOM TOKENIZERS |
+** |
+** Applications may also register custom tokenizer types. A tokenizer |
+** is registered by providing fts5 with a populated instance of the |
+** following structure. All structure methods must be defined, setting |
+** any member of the fts5_tokenizer struct to NULL leads to undefined |
+** behaviour. The structure methods are expected to function as follows: |
+** |
+** xCreate: |
+** This function is used to allocate and initialize a tokenizer instance. |
+** A tokenizer instance is required to actually tokenize text. |
+** |
+** The first argument passed to this function is a copy of the (void*) |
+** pointer provided by the application when the fts5_tokenizer object |
+** was registered with FTS5 (the third argument to xCreateTokenizer()). |
+** The second and third arguments are an array of nul-terminated strings |
+** containing the tokenizer arguments, if any, specified following the |
+** tokenizer name as part of the CREATE VIRTUAL TABLE statement used |
+** to create the FTS5 table. |
+** |
+** The final argument is an output variable. If successful, (*ppOut) |
+** should be set to point to the new tokenizer handle and SQLITE_OK |
+** returned. If an error occurs, some value other than SQLITE_OK should |
+** be returned. In this case, fts5 assumes that the final value of *ppOut |
+** is undefined. |
+** |
+** xDelete: |
+** This function is invoked to delete a tokenizer handle previously |
+** allocated using xCreate(). Fts5 guarantees that this function will |
+** be invoked exactly once for each successful call to xCreate(). |
+** |
+** xTokenize: |
+** This function is expected to tokenize the nText byte string indicated |
+** by argument pText. pText may or may not be nul-terminated. The first |
+** argument passed to this function is a pointer to an Fts5Tokenizer object |
+** returned by an earlier call to xCreate(). |
+** |
+** The second argument indicates the reason that FTS5 is requesting |
+** tokenization of the supplied text. This is always one of the following |
+** four values: |
+** |
+** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into |
+** or removed from the FTS table. The tokenizer is being invoked to |
+** determine the set of tokens to add to (or delete from) the |
+** FTS index. |
+** |
+** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed |
+** against the FTS index. The tokenizer is being called to tokenize |
+** a bareword or quoted string specified as part of the query. |
+** |
+** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as |
+** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is |
+** followed by a "*" character, indicating that the last token |
+** returned by the tokenizer will be treated as a token prefix. |
+** |
+** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to |
+** satisfy an fts5_api.xTokenize() request made by an auxiliary |
+** function. Or an fts5_api.xColumnSize() request made by the same |
+** on a columnsize=0 database. |
+** </ul> |
+** |
+** For each token in the input string, the supplied callback xToken() must |
+** be invoked. The first argument to it should be a copy of the pointer |
+** passed as the second argument to xTokenize(). The third and fourth |
+** arguments are a pointer to a buffer containing the token text, and the |
+** size of the token in bytes. The 4th and 5th arguments are the byte offsets |
+** of the first byte of and first byte immediately following the text from |
+** which the token is derived within the input. |
+** |
+** The second argument passed to the xToken() callback ("tflags") should |
+** normally be set to 0. The exception is if the tokenizer supports |
+** synonyms. In this case see the discussion below for details. |
+** |
+** FTS5 assumes the xToken() callback is invoked for each token in the |
+** order that they occur within the input text. |
+** |
+** If an xToken() callback returns any value other than SQLITE_OK, then |
+** the tokenization should be abandoned and the xTokenize() method should |
+** immediately return a copy of the xToken() return value. Or, if the |
+** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, |
+** if an error occurs with the xTokenize() implementation itself, it |
+** may abandon the tokenization and return any error code other than |
+** SQLITE_OK or SQLITE_DONE. |
+** |
+** SYNONYM SUPPORT |
+** |
+** Custom tokenizers may also support synonyms. Consider a case in which a |
+** user wishes to query for a phrase such as "first place". Using the |
+** built-in tokenizers, the FTS5 query 'first + place' will match instances |
+** of "first place" within the document set, but not alternative forms |
+** such as "1st place". In some applications, it would be better to match |
+** all instances of "first place" or "1st place" regardless of which form |
+** the user specified in the MATCH query text. |
+** |
+** There are several ways to approach this in FTS5: |
+** |
+** <ol><li> By mapping all synonyms to a single token. In this case, the |
+** In the above example, this means that the tokenizer returns the |
+** same token for inputs "first" and "1st". Say that token is in |
+** fact "first", so that when the user inserts the document "I won |
+** 1st place" entries are added to the index for tokens "i", "won", |
+** "first" and "place". If the user then queries for '1st + place', |
+** the tokenizer substitutes "first" for "1st" and the query works |
+** as expected. |
+** |
+** <li> By adding multiple synonyms for a single term to the FTS index. |
+** In this case, when tokenizing query text, the tokenizer may |
+** provide multiple synonyms for a single term within the document. |
+** FTS5 then queries the index for each synonym individually. For |
+** example, faced with the query: |
+** |
+** <codeblock> |
+** ... MATCH 'first place'</codeblock> |
+** |
+** the tokenizer offers both "1st" and "first" as synonyms for the |
+** first token in the MATCH query and FTS5 effectively runs a query |
+** similar to: |
+** |
+** <codeblock> |
+** ... MATCH '(first OR 1st) place'</codeblock> |
+** |
+** except that, for the purposes of auxiliary functions, the query |
+** still appears to contain just two phrases - "(first OR 1st)" |
+** being treated as a single phrase. |
+** |
+** <li> By adding multiple synonyms for a single term to the FTS index. |
+** Using this method, when tokenizing document text, the tokenizer |
+** provides multiple synonyms for each token. So that when a |
+** document such as "I won first place" is tokenized, entries are |
+** added to the FTS index for "i", "won", "first", "1st" and |
+** "place". |
+** |
+** This way, even if the tokenizer does not provide synonyms |
+** when tokenizing query text (it should not - to do would be |
+** inefficient), it doesn't matter if the user queries for |
+** 'first + place' or '1st + place', as there are entires in the |
+** FTS index corresponding to both forms of the first token. |
+** </ol> |
+** |
+** Whether it is parsing document or query text, any call to xToken that |
+** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit |
+** is considered to supply a synonym for the previous token. For example, |
+** when parsing the document "I won first place", a tokenizer that supports |
+** synonyms would call xToken() 5 times, as follows: |
+** |
+** <codeblock> |
+** xToken(pCtx, 0, "i", 1, 0, 1); |
+** xToken(pCtx, 0, "won", 3, 2, 5); |
+** xToken(pCtx, 0, "first", 5, 6, 11); |
+** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); |
+** xToken(pCtx, 0, "place", 5, 12, 17); |
+**</codeblock> |
+** |
+** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time |
+** xToken() is called. Multiple synonyms may be specified for a single token |
+** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. |
+** There is no limit to the number of synonyms that may be provided for a |
+** single token. |
+** |
+** In many cases, method (1) above is the best approach. It does not add |
+** extra data to the FTS index or require FTS5 to query for multiple terms, |
+** so it is efficient in terms of disk space and query speed. However, it |
+** does not support prefix queries very well. If, as suggested above, the |
+** token "first" is subsituted for "1st" by the tokenizer, then the query: |
+** |
+** <codeblock> |
+** ... MATCH '1s*'</codeblock> |
+** |
+** will not match documents that contain the token "1st" (as the tokenizer |
+** will probably not map "1s" to any prefix of "first"). |
+** |
+** For full prefix support, method (3) may be preferred. In this case, |
+** because the index contains entries for both "first" and "1st", prefix |
+** queries such as 'fi*' or '1s*' will match correctly. However, because |
+** extra entries are added to the FTS index, this method uses more space |
+** within the database. |
+** |
+** Method (2) offers a midpoint between (1) and (3). Using this method, |
+** a query such as '1s*' will match documents that contain the literal |
+** token "1st", but not "first" (assuming the tokenizer is not able to |
+** provide synonyms for prefixes). However, a non-prefix query like '1st' |
+** will match against "1st" and "first". This method does not require |
+** extra disk space, as no extra entries are added to the FTS index. |
+** On the other hand, it may require more CPU cycles to run MATCH queries, |
+** as separate queries of the FTS index are required for each synonym. |
+** |
+** When using methods (2) or (3), it is important that the tokenizer only |
+** provide synonyms when tokenizing document text (method (2)) or query |
+** text (method (3)), not both. Doing so will not cause any errors, but is |
+** inefficient. |
+*/ |
+typedef struct Fts5Tokenizer Fts5Tokenizer; |
+typedef struct fts5_tokenizer fts5_tokenizer; |
+struct fts5_tokenizer { |
+ int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); |
+ void (*xDelete)(Fts5Tokenizer*); |
+ int (*xTokenize)(Fts5Tokenizer*, |
+ void *pCtx, |
+ int flags, /* Mask of FTS5_TOKENIZE_* flags */ |
+ const char *pText, int nText, |
+ int (*xToken)( |
+ void *pCtx, /* Copy of 2nd argument to xTokenize() */ |
+ int tflags, /* Mask of FTS5_TOKEN_* flags */ |
+ const char *pToken, /* Pointer to buffer containing token */ |
+ int nToken, /* Size of token in bytes */ |
+ int iStart, /* Byte offset of token within input text */ |
+ int iEnd /* Byte offset of end of token within input text */ |
+ ) |
+ ); |
+}; |
+ |
+/* Flags that may be passed as the third argument to xTokenize() */ |
+#define FTS5_TOKENIZE_QUERY 0x0001 |
+#define FTS5_TOKENIZE_PREFIX 0x0002 |
+#define FTS5_TOKENIZE_DOCUMENT 0x0004 |
+#define FTS5_TOKENIZE_AUX 0x0008 |
+ |
+/* Flags that may be passed by the tokenizer implementation back to FTS5 |
+** as the third argument to the supplied xToken callback. */ |
+#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ |
+ |
+/* |
+** END OF CUSTOM TOKENIZERS |
+*************************************************************************/ |
+ |
+/************************************************************************* |
+** FTS5 EXTENSION REGISTRATION API |
+*/ |
+typedef struct fts5_api fts5_api; |
+struct fts5_api { |
+ int iVersion; /* Currently always set to 2 */ |
+ |
+ /* Create a new tokenizer */ |
+ int (*xCreateTokenizer)( |
+ fts5_api *pApi, |
+ const char *zName, |
+ void *pContext, |
+ fts5_tokenizer *pTokenizer, |
+ void (*xDestroy)(void*) |
+ ); |
+ |
+ /* Find an existing tokenizer */ |
+ int (*xFindTokenizer)( |
+ fts5_api *pApi, |
+ const char *zName, |
+ void **ppContext, |
+ fts5_tokenizer *pTokenizer |
+ ); |
+ |
+ /* Create a new auxiliary function */ |
+ int (*xCreateFunction)( |
+ fts5_api *pApi, |
+ const char *zName, |
+ void *pContext, |
+ fts5_extension_function xFunction, |
+ void (*xDestroy)(void*) |
+ ); |
+}; |
+ |
+/* |
+** END OF REGISTRATION API |
+*************************************************************************/ |
+ |
+#if 0 |
+} /* end of the 'extern "C"' block */ |
+#endif |
+ |
+#endif /* _FTS5_H */ |
+ |
+/* |
+** 2014 May 31 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+*/ |
+#ifndef _FTS5INT_H |
+#define _FTS5INT_H |
+ |
+/* #include "fts5.h" */ |
+/* #include "sqlite3ext.h" */ |
+SQLITE_EXTENSION_INIT1 |
+ |
+/* #include <string.h> */ |
+/* #include <assert.h> */ |
+ |
+#ifndef SQLITE_AMALGAMATION |
+ |
+typedef unsigned char u8; |
+typedef unsigned int u32; |
+typedef unsigned short u16; |
+typedef short i16; |
+typedef sqlite3_int64 i64; |
+typedef sqlite3_uint64 u64; |
+ |
+#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) |
+ |
+#define testcase(x) |
+#define ALWAYS(x) 1 |
+#define NEVER(x) 0 |
+ |
+#define MIN(x,y) (((x) < (y)) ? (x) : (y)) |
+#define MAX(x,y) (((x) > (y)) ? (x) : (y)) |
+ |
+/* |
+** Constants for the largest and smallest possible 64-bit signed integers. |
+*/ |
+# define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) |
+# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) |
+ |
+#endif |
+ |
+/* Truncate very long tokens to this many bytes. Hard limit is |
+** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset |
+** field that occurs at the start of each leaf page (see fts5_index.c). */ |
+#define FTS5_MAX_TOKEN_SIZE 32768 |
+ |
+/* |
+** Maximum number of prefix indexes on single FTS5 table. This must be |
+** less than 32. If it is set to anything large than that, an #error |
+** directive in fts5_index.c will cause the build to fail. |
+*/ |
+#define FTS5_MAX_PREFIX_INDEXES 31 |
+ |
+#define FTS5_DEFAULT_NEARDIST 10 |
+#define FTS5_DEFAULT_RANK "bm25" |
+ |
+/* Name of rank and rowid columns */ |
+#define FTS5_RANK_NAME "rank" |
+#define FTS5_ROWID_NAME "rowid" |
+ |
+#ifdef SQLITE_DEBUG |
+# define FTS5_CORRUPT sqlite3Fts5Corrupt() |
+static int sqlite3Fts5Corrupt(void); |
+#else |
+# define FTS5_CORRUPT SQLITE_CORRUPT_VTAB |
+#endif |
+ |
+/* |
+** The assert_nc() macro is similar to the assert() macro, except that it |
+** is used for assert() conditions that are true only if it can be |
+** guranteed that the database is not corrupt. |
+*/ |
+#ifdef SQLITE_DEBUG |
+SQLITE_API extern int sqlite3_fts5_may_be_corrupt; |
+# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x)) |
+#else |
+# define assert_nc(x) assert(x) |
+#endif |
+ |
+/* Mark a function parameter as unused, to suppress nuisance compiler |
+** warnings. */ |
+#ifndef UNUSED_PARAM |
+# define UNUSED_PARAM(X) (void)(X) |
+#endif |
+ |
+#ifndef UNUSED_PARAM2 |
+# define UNUSED_PARAM2(X, Y) (void)(X), (void)(Y) |
+#endif |
+ |
+typedef struct Fts5Global Fts5Global; |
+typedef struct Fts5Colset Fts5Colset; |
+ |
+/* If a NEAR() clump or phrase may only match a specific set of columns, |
+** then an object of the following type is used to record the set of columns. |
+** Each entry in the aiCol[] array is a column that may be matched. |
+** |
+** This object is used by fts5_expr.c and fts5_index.c. |
+*/ |
+struct Fts5Colset { |
+ int nCol; |
+ int aiCol[1]; |
+}; |
+ |
+ |
+ |
+/************************************************************************** |
+** Interface to code in fts5_config.c. fts5_config.c contains contains code |
+** to parse the arguments passed to the CREATE VIRTUAL TABLE statement. |
+*/ |
+ |
+typedef struct Fts5Config Fts5Config; |
+ |
+/* |
+** An instance of the following structure encodes all information that can |
+** be gleaned from the CREATE VIRTUAL TABLE statement. |
+** |
+** And all information loaded from the %_config table. |
+** |
+** nAutomerge: |
+** The minimum number of segments that an auto-merge operation should |
+** attempt to merge together. A value of 1 sets the object to use the |
+** compile time default. Zero disables auto-merge altogether. |
+** |
+** zContent: |
+** |
+** zContentRowid: |
+** The value of the content_rowid= option, if one was specified. Or |
+** the string "rowid" otherwise. This text is not quoted - if it is |
+** used as part of an SQL statement it needs to be quoted appropriately. |
+** |
+** zContentExprlist: |
+** |
+** pzErrmsg: |
+** This exists in order to allow the fts5_index.c module to return a |
+** decent error message if it encounters a file-format version it does |
+** not understand. |
+** |
+** bColumnsize: |
+** True if the %_docsize table is created. |
+** |
+** bPrefixIndex: |
+** This is only used for debugging. If set to false, any prefix indexes |
+** are ignored. This value is configured using: |
+** |
+** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex); |
+** |
+*/ |
+struct Fts5Config { |
+ sqlite3 *db; /* Database handle */ |
+ char *zDb; /* Database holding FTS index (e.g. "main") */ |
+ char *zName; /* Name of FTS index */ |
+ int nCol; /* Number of columns */ |
+ char **azCol; /* Column names */ |
+ u8 *abUnindexed; /* True for unindexed columns */ |
+ int nPrefix; /* Number of prefix indexes */ |
+ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ |
+ int eContent; /* An FTS5_CONTENT value */ |
+ char *zContent; /* content table */ |
+ char *zContentRowid; /* "content_rowid=" option value */ |
+ int bColumnsize; /* "columnsize=" option value (dflt==1) */ |
+ int eDetail; /* FTS5_DETAIL_XXX value */ |
+ char *zContentExprlist; |
+ Fts5Tokenizer *pTok; |
+ fts5_tokenizer *pTokApi; |
+ |
+ /* Values loaded from the %_config table */ |
+ int iCookie; /* Incremented when %_config is modified */ |
+ int pgsz; /* Approximate page size used in %_data */ |
+ int nAutomerge; /* 'automerge' setting */ |
+ int nCrisisMerge; /* Maximum allowed segments per level */ |
+ int nUsermerge; /* 'usermerge' setting */ |
+ int nHashSize; /* Bytes of memory for in-memory hash */ |
+ char *zRank; /* Name of rank function */ |
+ char *zRankArgs; /* Arguments to rank function */ |
+ |
+ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ |
+ char **pzErrmsg; |
+ |
+#ifdef SQLITE_DEBUG |
+ int bPrefixIndex; /* True to use prefix-indexes */ |
+#endif |
+}; |
+ |
+/* Current expected value of %_config table 'version' field */ |
+#define FTS5_CURRENT_VERSION 4 |
+ |
+#define FTS5_CONTENT_NORMAL 0 |
+#define FTS5_CONTENT_NONE 1 |
+#define FTS5_CONTENT_EXTERNAL 2 |
+ |
+#define FTS5_DETAIL_FULL 0 |
+#define FTS5_DETAIL_NONE 1 |
+#define FTS5_DETAIL_COLUMNS 2 |
+ |
+ |
+ |
+static int sqlite3Fts5ConfigParse( |
+ Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** |
+); |
+static void sqlite3Fts5ConfigFree(Fts5Config*); |
+ |
+static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig); |
+ |
+static int sqlite3Fts5Tokenize( |
+ Fts5Config *pConfig, /* FTS5 Configuration object */ |
+ int flags, /* FTS5_TOKENIZE_* flags */ |
+ const char *pText, int nText, /* Text to tokenize */ |
+ void *pCtx, /* Context passed to xToken() */ |
+ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ |
+); |
+ |
+static void sqlite3Fts5Dequote(char *z); |
+ |
+/* Load the contents of the %_config table */ |
+static int sqlite3Fts5ConfigLoad(Fts5Config*, int); |
+ |
+/* Set the value of a single config attribute */ |
+static int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*); |
+ |
+static int sqlite3Fts5ConfigParseRank(const char*, char**, char**); |
+ |
+/* |
+** End of interface to code in fts5_config.c. |
+**************************************************************************/ |
+ |
+/************************************************************************** |
+** Interface to code in fts5_buffer.c. |
+*/ |
+ |
+/* |
+** Buffer object for the incremental building of string data. |
+*/ |
+typedef struct Fts5Buffer Fts5Buffer; |
+struct Fts5Buffer { |
+ u8 *p; |
+ int n; |
+ int nSpace; |
+}; |
+ |
+static int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32); |
+static void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64); |
+static void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*); |
+static void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); |
+static void sqlite3Fts5BufferFree(Fts5Buffer*); |
+static void sqlite3Fts5BufferZero(Fts5Buffer*); |
+static void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); |
+static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); |
+ |
+static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); |
+ |
+#define fts5BufferZero(x) sqlite3Fts5BufferZero(x) |
+#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) |
+#define fts5BufferFree(a) sqlite3Fts5BufferFree(a) |
+#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) |
+#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) |
+ |
+#define fts5BufferGrow(pRc,pBuf,nn) ( \ |
+ (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \ |
+ sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \ |
+) |
+ |
+/* Write and decode big-endian 32-bit integer values */ |
+static void sqlite3Fts5Put32(u8*, int); |
+static int sqlite3Fts5Get32(const u8*); |
+ |
+#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32) |
+#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF) |
+ |
+typedef struct Fts5PoslistReader Fts5PoslistReader; |
+struct Fts5PoslistReader { |
+ /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */ |
+ const u8 *a; /* Position list to iterate through */ |
+ int n; /* Size of buffer at a[] in bytes */ |
+ int i; /* Current offset in a[] */ |
+ |
+ u8 bFlag; /* For client use (any custom purpose) */ |
+ |
+ /* Output variables */ |
+ u8 bEof; /* Set to true at EOF */ |
+ i64 iPos; /* (iCol<<32) + iPos */ |
+}; |
+static int sqlite3Fts5PoslistReaderInit( |
+ const u8 *a, int n, /* Poslist buffer to iterate through */ |
+ Fts5PoslistReader *pIter /* Iterator object to initialize */ |
+); |
+static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*); |
+ |
+typedef struct Fts5PoslistWriter Fts5PoslistWriter; |
+struct Fts5PoslistWriter { |
+ i64 iPrev; |
+}; |
+static int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64); |
+static void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64); |
+ |
+static int sqlite3Fts5PoslistNext64( |
+ const u8 *a, int n, /* Buffer containing poslist */ |
+ int *pi, /* IN/OUT: Offset within a[] */ |
+ i64 *piOff /* IN/OUT: Current offset */ |
+); |
+ |
+/* Malloc utility */ |
+static void *sqlite3Fts5MallocZero(int *pRc, int nByte); |
+static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); |
+ |
+/* Character set tests (like isspace(), isalpha() etc.) */ |
+static int sqlite3Fts5IsBareword(char t); |
+ |
+ |
+/* Bucket of terms object used by the integrity-check in offsets=0 mode. */ |
+typedef struct Fts5Termset Fts5Termset; |
+static int sqlite3Fts5TermsetNew(Fts5Termset**); |
+static int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent); |
+static void sqlite3Fts5TermsetFree(Fts5Termset*); |
+ |
+/* |
+** End of interface to code in fts5_buffer.c. |
+**************************************************************************/ |
+ |
+/************************************************************************** |
+** Interface to code in fts5_index.c. fts5_index.c contains contains code |
+** to access the data stored in the %_data table. |
+*/ |
+ |
+typedef struct Fts5Index Fts5Index; |
+typedef struct Fts5IndexIter Fts5IndexIter; |
+ |
+struct Fts5IndexIter { |
+ i64 iRowid; |
+ const u8 *pData; |
+ int nData; |
+ u8 bEof; |
+}; |
+ |
+#define sqlite3Fts5IterEof(x) ((x)->bEof) |
+ |
+/* |
+** Values used as part of the flags argument passed to IndexQuery(). |
+*/ |
+#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ |
+#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ |
+#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ |
+#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ |
+ |
+/* The following are used internally by the fts5_index.c module. They are |
+** defined here only to make it easier to avoid clashes with the flags |
+** above. */ |
+#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 |
+#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 |
+ |
+/* |
+** Create/destroy an Fts5Index object. |
+*/ |
+static int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**); |
+static int sqlite3Fts5IndexClose(Fts5Index *p); |
+ |
+/* |
+** Return a simple checksum value based on the arguments. |
+*/ |
+static u64 sqlite3Fts5IndexEntryCksum( |
+ i64 iRowid, |
+ int iCol, |
+ int iPos, |
+ int iIdx, |
+ const char *pTerm, |
+ int nTerm |
+); |
+ |
+/* |
+** Argument p points to a buffer containing utf-8 text that is n bytes in |
+** size. Return the number of bytes in the nChar character prefix of the |
+** buffer, or 0 if there are less than nChar characters in total. |
+*/ |
+static int sqlite3Fts5IndexCharlenToBytelen( |
+ const char *p, |
+ int nByte, |
+ int nChar |
+); |
+ |
+/* |
+** Open a new iterator to iterate though all rowids that match the |
+** specified token or token prefix. |
+*/ |
+static int sqlite3Fts5IndexQuery( |
+ Fts5Index *p, /* FTS index to query */ |
+ const char *pToken, int nToken, /* Token (or prefix) to query for */ |
+ int flags, /* Mask of FTS5INDEX_QUERY_X flags */ |
+ Fts5Colset *pColset, /* Match these columns only */ |
+ Fts5IndexIter **ppIter /* OUT: New iterator object */ |
+); |
+ |
+/* |
+** The various operations on open token or token prefix iterators opened |
+** using sqlite3Fts5IndexQuery(). |
+*/ |
+static int sqlite3Fts5IterNext(Fts5IndexIter*); |
+static int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); |
+ |
+/* |
+** Close an iterator opened by sqlite3Fts5IndexQuery(). |
+*/ |
+static void sqlite3Fts5IterClose(Fts5IndexIter*); |
+ |
+/* |
+** This interface is used by the fts5vocab module. |
+*/ |
+static const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*); |
+static int sqlite3Fts5IterNextScan(Fts5IndexIter*); |
+ |
+ |
+/* |
+** Insert or remove data to or from the index. Each time a document is |
+** added to or removed from the index, this function is called one or more |
+** times. |
+** |
+** For an insert, it must be called once for each token in the new document. |
+** If the operation is a delete, it must be called (at least) once for each |
+** unique token in the document with an iCol value less than zero. The iPos |
+** argument is ignored for a delete. |
+*/ |
+static int sqlite3Fts5IndexWrite( |
+ Fts5Index *p, /* Index to write to */ |
+ int iCol, /* Column token appears in (-ve -> delete) */ |
+ int iPos, /* Position of token within column */ |
+ const char *pToken, int nToken /* Token to add or remove to or from index */ |
+); |
+ |
+/* |
+** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to |
+** document iDocid. |
+*/ |
+static int sqlite3Fts5IndexBeginWrite( |
+ Fts5Index *p, /* Index to write to */ |
+ int bDelete, /* True if current operation is a delete */ |
+ i64 iDocid /* Docid to add or remove data from */ |
+); |
+ |
+/* |
+** Flush any data stored in the in-memory hash tables to the database. |
+** If the bCommit flag is true, also close any open blob handles. |
+*/ |
+static int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit); |
+ |
+/* |
+** Discard any data stored in the in-memory hash tables. Do not write it |
+** to the database. Additionally, assume that the contents of the %_data |
+** table may have changed on disk. So any in-memory caches of %_data |
+** records must be invalidated. |
+*/ |
+static int sqlite3Fts5IndexRollback(Fts5Index *p); |
+ |
+/* |
+** Get or set the "averages" values. |
+*/ |
+static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); |
+static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); |
+ |
+/* |
+** Functions called by the storage module as part of integrity-check. |
+*/ |
+static int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum); |
+ |
+/* |
+** Called during virtual module initialization to register UDF |
+** fts5_decode() with SQLite |
+*/ |
+static int sqlite3Fts5IndexInit(sqlite3*); |
+ |
+static int sqlite3Fts5IndexSetCookie(Fts5Index*, int); |
+ |
+/* |
+** Return the total number of entries read from the %_data table by |
+** this connection since it was created. |
+*/ |
+static int sqlite3Fts5IndexReads(Fts5Index *p); |
+ |
+static int sqlite3Fts5IndexReinit(Fts5Index *p); |
+static int sqlite3Fts5IndexOptimize(Fts5Index *p); |
+static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); |
+static int sqlite3Fts5IndexReset(Fts5Index *p); |
+ |
+static int sqlite3Fts5IndexLoadConfig(Fts5Index *p); |
+ |
+/* |
+** End of interface to code in fts5_index.c. |
+**************************************************************************/ |
+ |
+/************************************************************************** |
+** Interface to code in fts5_varint.c. |
+*/ |
+static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v); |
+static int sqlite3Fts5GetVarintLen(u32 iVal); |
+static u8 sqlite3Fts5GetVarint(const unsigned char*, u64*); |
+static int sqlite3Fts5PutVarint(unsigned char *p, u64 v); |
+ |
+#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b) |
+#define fts5GetVarint sqlite3Fts5GetVarint |
+ |
+#define fts5FastGetVarint32(a, iOff, nVal) { \ |
+ nVal = (a)[iOff++]; \ |
+ if( nVal & 0x80 ){ \ |
+ iOff--; \ |
+ iOff += fts5GetVarint32(&(a)[iOff], nVal); \ |
+ } \ |
+} |
+ |
+ |
+/* |
+** End of interface to code in fts5_varint.c. |
+**************************************************************************/ |
+ |
+ |
+/************************************************************************** |
+** Interface to code in fts5.c. |
+*/ |
+ |
+static int sqlite3Fts5GetTokenizer( |
+ Fts5Global*, |
+ const char **azArg, |
+ int nArg, |
+ Fts5Tokenizer**, |
+ fts5_tokenizer**, |
+ char **pzErr |
+); |
+ |
+static Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **); |
+ |
+/* |
+** End of interface to code in fts5.c. |
+**************************************************************************/ |
+ |
+/************************************************************************** |
+** Interface to code in fts5_hash.c. |
+*/ |
+typedef struct Fts5Hash Fts5Hash; |
+ |
+/* |
+** Create a hash table, free a hash table. |
+*/ |
+static int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize); |
+static void sqlite3Fts5HashFree(Fts5Hash*); |
+ |
+static int sqlite3Fts5HashWrite( |
+ Fts5Hash*, |
+ i64 iRowid, /* Rowid for this entry */ |
+ int iCol, /* Column token appears in (-ve -> delete) */ |
+ int iPos, /* Position of token within column */ |
+ char bByte, |
+ const char *pToken, int nToken /* Token to add or remove to or from index */ |
+); |
+ |
+/* |
+** Empty (but do not delete) a hash table. |
+*/ |
+static void sqlite3Fts5HashClear(Fts5Hash*); |
+ |
+static int sqlite3Fts5HashQuery( |
+ Fts5Hash*, /* Hash table to query */ |
+ const char *pTerm, int nTerm, /* Query term */ |
+ const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ |
+ int *pnDoclist /* OUT: Size of doclist in bytes */ |
+); |
+ |
+static int sqlite3Fts5HashScanInit( |
+ Fts5Hash*, /* Hash table to query */ |
+ const char *pTerm, int nTerm /* Query prefix */ |
+); |
+static void sqlite3Fts5HashScanNext(Fts5Hash*); |
+static int sqlite3Fts5HashScanEof(Fts5Hash*); |
+static void sqlite3Fts5HashScanEntry(Fts5Hash *, |
+ const char **pzTerm, /* OUT: term (nul-terminated) */ |
+ const u8 **ppDoclist, /* OUT: pointer to doclist */ |
+ int *pnDoclist /* OUT: size of doclist in bytes */ |
+); |
+ |
+ |
+/* |
+** End of interface to code in fts5_hash.c. |
+**************************************************************************/ |
+ |
+/************************************************************************** |
+** Interface to code in fts5_storage.c. fts5_storage.c contains contains |
+** code to access the data stored in the %_content and %_docsize tables. |
+*/ |
+ |
+#define FTS5_STMT_SCAN_ASC 0 /* SELECT rowid, * FROM ... ORDER BY 1 ASC */ |
+#define FTS5_STMT_SCAN_DESC 1 /* SELECT rowid, * FROM ... ORDER BY 1 DESC */ |
+#define FTS5_STMT_LOOKUP 2 /* SELECT rowid, * FROM ... WHERE rowid=? */ |
+ |
+typedef struct Fts5Storage Fts5Storage; |
+ |
+static int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**); |
+static int sqlite3Fts5StorageClose(Fts5Storage *p); |
+static int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); |
+ |
+static int sqlite3Fts5DropAll(Fts5Config*); |
+static int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); |
+ |
+static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**); |
+static int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); |
+static int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); |
+ |
+static int sqlite3Fts5StorageIntegrity(Fts5Storage *p); |
+ |
+static int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**); |
+static void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); |
+ |
+static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); |
+static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg); |
+static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow); |
+ |
+static int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit); |
+static int sqlite3Fts5StorageRollback(Fts5Storage *p); |
+ |
+static int sqlite3Fts5StorageConfigValue( |
+ Fts5Storage *p, const char*, sqlite3_value*, int |
+); |
+ |
+static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); |
+static int sqlite3Fts5StorageRebuild(Fts5Storage *p); |
+static int sqlite3Fts5StorageOptimize(Fts5Storage *p); |
+static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); |
+static int sqlite3Fts5StorageReset(Fts5Storage *p); |
+ |
+/* |
+** End of interface to code in fts5_storage.c. |
+**************************************************************************/ |
+ |
+ |
+/************************************************************************** |
+** Interface to code in fts5_expr.c. |
+*/ |
+typedef struct Fts5Expr Fts5Expr; |
+typedef struct Fts5ExprNode Fts5ExprNode; |
+typedef struct Fts5Parse Fts5Parse; |
+typedef struct Fts5Token Fts5Token; |
+typedef struct Fts5ExprPhrase Fts5ExprPhrase; |
+typedef struct Fts5ExprNearset Fts5ExprNearset; |
+ |
+struct Fts5Token { |
+ const char *p; /* Token text (not NULL terminated) */ |
+ int n; /* Size of buffer p in bytes */ |
+}; |
+ |
+/* Parse a MATCH expression. */ |
+static int sqlite3Fts5ExprNew( |
+ Fts5Config *pConfig, |
+ const char *zExpr, |
+ Fts5Expr **ppNew, |
+ char **pzErr |
+); |
+ |
+/* |
+** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc); |
+** rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr); |
+** rc = sqlite3Fts5ExprNext(pExpr) |
+** ){ |
+** // The document with rowid iRowid matches the expression! |
+** i64 iRowid = sqlite3Fts5ExprRowid(pExpr); |
+** } |
+*/ |
+static int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc); |
+static int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax); |
+static int sqlite3Fts5ExprEof(Fts5Expr*); |
+static i64 sqlite3Fts5ExprRowid(Fts5Expr*); |
+ |
+static void sqlite3Fts5ExprFree(Fts5Expr*); |
+ |
+/* Called during startup to register a UDF with SQLite */ |
+static int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); |
+ |
+static int sqlite3Fts5ExprPhraseCount(Fts5Expr*); |
+static int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); |
+static int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); |
+ |
+typedef struct Fts5PoslistPopulator Fts5PoslistPopulator; |
+static Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int); |
+static int sqlite3Fts5ExprPopulatePoslists( |
+ Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int |
+); |
+static void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); |
+ |
+static int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); |
+ |
+static int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); |
+ |
+/******************************************* |
+** The fts5_expr.c API above this point is used by the other hand-written |
+** C code in this module. The interfaces below this point are called by |
+** the parser code in fts5parse.y. */ |
+ |
+static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...); |
+ |
+static Fts5ExprNode *sqlite3Fts5ParseNode( |
+ Fts5Parse *pParse, |
+ int eType, |
+ Fts5ExprNode *pLeft, |
+ Fts5ExprNode *pRight, |
+ Fts5ExprNearset *pNear |
+); |
+ |
+static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( |
+ Fts5Parse *pParse, |
+ Fts5ExprNode *pLeft, |
+ Fts5ExprNode *pRight |
+); |
+ |
+static Fts5ExprPhrase *sqlite3Fts5ParseTerm( |
+ Fts5Parse *pParse, |
+ Fts5ExprPhrase *pPhrase, |
+ Fts5Token *pToken, |
+ int bPrefix |
+); |
+ |
+static Fts5ExprNearset *sqlite3Fts5ParseNearset( |
+ Fts5Parse*, |
+ Fts5ExprNearset*, |
+ Fts5ExprPhrase* |
+); |
+ |
+static Fts5Colset *sqlite3Fts5ParseColset( |
+ Fts5Parse*, |
+ Fts5Colset*, |
+ Fts5Token * |
+); |
+ |
+static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*); |
+static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*); |
+static void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); |
+ |
+static void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); |
+static void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*); |
+static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*); |
+static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); |
+static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); |
+ |
+/* |
+** End of interface to code in fts5_expr.c. |
+**************************************************************************/ |
+ |
+ |
+ |
+/************************************************************************** |
+** Interface to code in fts5_aux.c. |
+*/ |
+ |
+static int sqlite3Fts5AuxInit(fts5_api*); |
+/* |
+** End of interface to code in fts5_aux.c. |
+**************************************************************************/ |
+ |
+/************************************************************************** |
+** Interface to code in fts5_tokenizer.c. |
+*/ |
+ |
+static int sqlite3Fts5TokenizerInit(fts5_api*); |
+/* |
+** End of interface to code in fts5_tokenizer.c. |
+**************************************************************************/ |
+ |
+/************************************************************************** |
+** Interface to code in fts5_vocab.c. |
+*/ |
+ |
+static int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*); |
+ |
+/* |
+** End of interface to code in fts5_vocab.c. |
+**************************************************************************/ |
+ |
+ |
+/************************************************************************** |
+** Interface to automatically generated code in fts5_unicode2.c. |
+*/ |
+static int sqlite3Fts5UnicodeIsalnum(int c); |
+static int sqlite3Fts5UnicodeIsdiacritic(int c); |
+static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); |
+/* |
+** End of interface to code in fts5_unicode2.c. |
+**************************************************************************/ |
+ |
+#endif |
+ |
+#define FTS5_OR 1 |
+#define FTS5_AND 2 |
+#define FTS5_NOT 3 |
+#define FTS5_TERM 4 |
+#define FTS5_COLON 5 |
+#define FTS5_LP 6 |
+#define FTS5_RP 7 |
+#define FTS5_MINUS 8 |
+#define FTS5_LCP 9 |
+#define FTS5_RCP 10 |
+#define FTS5_STRING 11 |
+#define FTS5_COMMA 12 |
+#define FTS5_PLUS 13 |
+#define FTS5_STAR 14 |
+ |
+/* |
+** 2000-05-29 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+************************************************************************* |
+** Driver template for the LEMON parser generator. |
+** |
+** The "lemon" program processes an LALR(1) input grammar file, then uses |
+** this template to construct a parser. The "lemon" program inserts text |
+** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the |
+** interstitial "-" characters) contained in this template is changed into |
+** the value of the %name directive from the grammar. Otherwise, the content |
+** of this template is copied straight through into the generate parser |
+** source file. |
+** |
+** The following is the concatenation of all %include directives from the |
+** input grammar file: |
+*/ |
+/* #include <stdio.h> */ |
+/************ Begin %include sections from the grammar ************************/ |
+ |
+/* #include "fts5Int.h" */ |
+/* #include "fts5parse.h" */ |
+ |
+/* |
+** Disable all error recovery processing in the parser push-down |
+** automaton. |
+*/ |
+#define fts5YYNOERRORRECOVERY 1 |
+ |
+/* |
+** Make fts5yytestcase() the same as testcase() |
+*/ |
+#define fts5yytestcase(X) testcase(X) |
+ |
+/* |
+** Indicate that sqlite3ParserFree() will never be called with a null |
+** pointer. |
+*/ |
+#define fts5YYPARSEFREENOTNULL 1 |
+ |
+/* |
+** Alternative datatype for the argument to the malloc() routine passed |
+** into sqlite3ParserAlloc(). The default is size_t. |
+*/ |
+#define fts5YYMALLOCARGTYPE u64 |
+ |
+/**************** End of %include directives **********************************/ |
+/* These constants specify the various numeric values for terminal symbols |
+** in a format understandable to "makeheaders". This section is blank unless |
+** "lemon" is run with the "-m" command-line option. |
+***************** Begin makeheaders token definitions *************************/ |
+/**************** End makeheaders token definitions ***************************/ |
+ |
+/* The next sections is a series of control #defines. |
+** various aspects of the generated parser. |
+** fts5YYCODETYPE is the data type used to store the integer codes |
+** that represent terminal and non-terminal symbols. |
+** "unsigned char" is used if there are fewer than |
+** 256 symbols. Larger types otherwise. |
+** fts5YYNOCODE is a number of type fts5YYCODETYPE that is not used for |
+** any terminal or nonterminal symbol. |
+** fts5YYFALLBACK If defined, this indicates that one or more tokens |
+** (also known as: "terminal symbols") have fall-back |
+** values which should be used if the original symbol |
+** would not parse. This permits keywords to sometimes |
+** be used as identifiers, for example. |
+** fts5YYACTIONTYPE is the data type used for "action codes" - numbers |
+** that indicate what to do in response to the next |
+** token. |
+** sqlite3Fts5ParserFTS5TOKENTYPE is the data type used for minor type for terminal |
+** symbols. Background: A "minor type" is a semantic |
+** value associated with a terminal or non-terminal |
+** symbols. For example, for an "ID" terminal symbol, |
+** the minor type might be the name of the identifier. |
+** Each non-terminal can have a different minor type. |
+** Terminal symbols all have the same minor type, though. |
+** This macros defines the minor type for terminal |
+** symbols. |
+** fts5YYMINORTYPE is the data type used for all minor types. |
+** This is typically a union of many types, one of |
+** which is sqlite3Fts5ParserFTS5TOKENTYPE. The entry in the union |
+** for terminal symbols is called "fts5yy0". |
+** fts5YYSTACKDEPTH is the maximum depth of the parser's stack. If |
+** zero the stack is dynamically sized using realloc() |
+** sqlite3Fts5ParserARG_SDECL A static variable declaration for the %extra_argument |
+** sqlite3Fts5ParserARG_PDECL A parameter declaration for the %extra_argument |
+** sqlite3Fts5ParserARG_STORE Code to store %extra_argument into fts5yypParser |
+** sqlite3Fts5ParserARG_FETCH Code to extract %extra_argument from fts5yypParser |
+** fts5YYERRORSYMBOL is the code number of the error symbol. If not |
+** defined, then do no error processing. |
+** fts5YYNSTATE the combined number of states. |
+** fts5YYNRULE the number of rules in the grammar |
+** fts5YY_MAX_SHIFT Maximum value for shift actions |
+** fts5YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions |
+** fts5YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions |
+** fts5YY_MIN_REDUCE Maximum value for reduce actions |
+** fts5YY_ERROR_ACTION The fts5yy_action[] code for syntax error |
+** fts5YY_ACCEPT_ACTION The fts5yy_action[] code for accept |
+** fts5YY_NO_ACTION The fts5yy_action[] code for no-op |
+*/ |
+#ifndef INTERFACE |
+# define INTERFACE 1 |
+#endif |
+/************* Begin control #defines *****************************************/ |
+#define fts5YYCODETYPE unsigned char |
+#define fts5YYNOCODE 28 |
+#define fts5YYACTIONTYPE unsigned char |
+#define sqlite3Fts5ParserFTS5TOKENTYPE Fts5Token |
+typedef union { |
+ int fts5yyinit; |
+ sqlite3Fts5ParserFTS5TOKENTYPE fts5yy0; |
+ int fts5yy4; |
+ Fts5Colset* fts5yy11; |
+ Fts5ExprNode* fts5yy24; |
+ Fts5ExprNearset* fts5yy46; |
+ Fts5ExprPhrase* fts5yy53; |
+} fts5YYMINORTYPE; |
+#ifndef fts5YYSTACKDEPTH |
+#define fts5YYSTACKDEPTH 100 |
+#endif |
+#define sqlite3Fts5ParserARG_SDECL Fts5Parse *pParse; |
+#define sqlite3Fts5ParserARG_PDECL ,Fts5Parse *pParse |
+#define sqlite3Fts5ParserARG_FETCH Fts5Parse *pParse = fts5yypParser->pParse |
+#define sqlite3Fts5ParserARG_STORE fts5yypParser->pParse = pParse |
+#define fts5YYNSTATE 29 |
+#define fts5YYNRULE 26 |
+#define fts5YY_MAX_SHIFT 28 |
+#define fts5YY_MIN_SHIFTREDUCE 45 |
+#define fts5YY_MAX_SHIFTREDUCE 70 |
+#define fts5YY_MIN_REDUCE 71 |
+#define fts5YY_MAX_REDUCE 96 |
+#define fts5YY_ERROR_ACTION 97 |
+#define fts5YY_ACCEPT_ACTION 98 |
+#define fts5YY_NO_ACTION 99 |
+/************* End control #defines *******************************************/ |
+ |
+/* Define the fts5yytestcase() macro to be a no-op if is not already defined |
+** otherwise. |
+** |
+** Applications can choose to define fts5yytestcase() in the %include section |
+** to a macro that can assist in verifying code coverage. For production |
+** code the fts5yytestcase() macro should be turned off. But it is useful |
+** for testing. |
+*/ |
+#ifndef fts5yytestcase |
+# define fts5yytestcase(X) |
+#endif |
+ |
+ |
+/* Next are the tables used to determine what action to take based on the |
+** current state and lookahead token. These tables are used to implement |
+** functions that take a state number and lookahead value and return an |
+** action integer. |
+** |
+** Suppose the action integer is N. Then the action is determined as |
+** follows |
+** |
+** 0 <= N <= fts5YY_MAX_SHIFT Shift N. That is, push the lookahead |
+** token onto the stack and goto state N. |
+** |
+** N between fts5YY_MIN_SHIFTREDUCE Shift to an arbitrary state then |
+** and fts5YY_MAX_SHIFTREDUCE reduce by rule N-fts5YY_MIN_SHIFTREDUCE. |
+** |
+** N between fts5YY_MIN_REDUCE Reduce by rule N-fts5YY_MIN_REDUCE |
+** and fts5YY_MAX_REDUCE |
+** |
+** N == fts5YY_ERROR_ACTION A syntax error has occurred. |
+** |
+** N == fts5YY_ACCEPT_ACTION The parser accepts its input. |
+** |
+** N == fts5YY_NO_ACTION No such action. Denotes unused |
+** slots in the fts5yy_action[] table. |
+** |
+** The action table is constructed as a single large table named fts5yy_action[]. |
+** Given state S and lookahead X, the action is computed as either: |
+** |
+** (A) N = fts5yy_action[ fts5yy_shift_ofst[S] + X ] |
+** (B) N = fts5yy_default[S] |
+** |
+** The (A) formula is preferred. The B formula is used instead if: |
+** (1) The fts5yy_shift_ofst[S]+X value is out of range, or |
+** (2) fts5yy_lookahead[fts5yy_shift_ofst[S]+X] is not equal to X, or |
+** (3) fts5yy_shift_ofst[S] equal fts5YY_SHIFT_USE_DFLT. |
+** (Implementation note: fts5YY_SHIFT_USE_DFLT is chosen so that |
+** fts5YY_SHIFT_USE_DFLT+X will be out of range for all possible lookaheads X. |
+** Hence only tests (1) and (2) need to be evaluated.) |
+** |
+** The formulas above are for computing the action when the lookahead is |
+** a terminal symbol. If the lookahead is a non-terminal (as occurs after |
+** a reduce action) then the fts5yy_reduce_ofst[] array is used in place of |
+** the fts5yy_shift_ofst[] array and fts5YY_REDUCE_USE_DFLT is used in place of |
+** fts5YY_SHIFT_USE_DFLT. |
+** |
+** The following are the tables generated in this section: |
+** |
+** fts5yy_action[] A single table containing all actions. |
+** fts5yy_lookahead[] A table containing the lookahead for each entry in |
+** fts5yy_action. Used to detect hash collisions. |
+** fts5yy_shift_ofst[] For each state, the offset into fts5yy_action for |
+** shifting terminals. |
+** fts5yy_reduce_ofst[] For each state, the offset into fts5yy_action for |
+** shifting non-terminals after a reduce. |
+** fts5yy_default[] Default action for each state. |
+** |
+*********** Begin parsing tables **********************************************/ |
+#define fts5YY_ACTTAB_COUNT (85) |
+static const fts5YYACTIONTYPE fts5yy_action[] = { |
+ /* 0 */ 98, 16, 51, 5, 53, 27, 83, 7, 26, 15, |
+ /* 10 */ 51, 5, 53, 27, 13, 69, 26, 48, 51, 5, |
+ /* 20 */ 53, 27, 19, 11, 26, 9, 20, 51, 5, 53, |
+ /* 30 */ 27, 13, 22, 26, 28, 51, 5, 53, 27, 68, |
+ /* 40 */ 1, 26, 19, 11, 17, 9, 52, 10, 53, 27, |
+ /* 50 */ 23, 24, 26, 54, 3, 4, 2, 26, 6, 21, |
+ /* 60 */ 49, 71, 3, 4, 2, 7, 56, 59, 55, 59, |
+ /* 70 */ 4, 2, 12, 69, 58, 60, 18, 67, 62, 69, |
+ /* 80 */ 25, 66, 8, 14, 2, |
+}; |
+static const fts5YYCODETYPE fts5yy_lookahead[] = { |
+ /* 0 */ 16, 17, 18, 19, 20, 21, 5, 6, 24, 17, |
+ /* 10 */ 18, 19, 20, 21, 11, 14, 24, 17, 18, 19, |
+ /* 20 */ 20, 21, 8, 9, 24, 11, 17, 18, 19, 20, |
+ /* 30 */ 21, 11, 12, 24, 17, 18, 19, 20, 21, 26, |
+ /* 40 */ 6, 24, 8, 9, 22, 11, 18, 11, 20, 21, |
+ /* 50 */ 24, 25, 24, 20, 1, 2, 3, 24, 23, 24, |
+ /* 60 */ 7, 0, 1, 2, 3, 6, 10, 11, 10, 11, |
+ /* 70 */ 2, 3, 9, 14, 11, 11, 22, 26, 7, 14, |
+ /* 80 */ 13, 11, 5, 11, 3, |
+}; |
+#define fts5YY_SHIFT_USE_DFLT (85) |
+#define fts5YY_SHIFT_COUNT (28) |
+#define fts5YY_SHIFT_MIN (0) |
+#define fts5YY_SHIFT_MAX (81) |
+static const unsigned char fts5yy_shift_ofst[] = { |
+ /* 0 */ 34, 34, 34, 34, 34, 14, 20, 3, 36, 1, |
+ /* 10 */ 59, 64, 64, 65, 65, 53, 61, 56, 58, 63, |
+ /* 20 */ 68, 67, 70, 67, 71, 72, 67, 77, 81, |
+}; |
+#define fts5YY_REDUCE_USE_DFLT (-17) |
+#define fts5YY_REDUCE_COUNT (14) |
+#define fts5YY_REDUCE_MIN (-16) |
+#define fts5YY_REDUCE_MAX (54) |
+static const signed char fts5yy_reduce_ofst[] = { |
+ /* 0 */ -16, -8, 0, 9, 17, 28, 26, 35, 33, 13, |
+ /* 10 */ 13, 22, 54, 13, 51, |
+}; |
+static const fts5YYACTIONTYPE fts5yy_default[] = { |
+ /* 0 */ 97, 97, 97, 97, 97, 76, 91, 97, 97, 96, |
+ /* 10 */ 96, 97, 97, 96, 96, 97, 97, 97, 97, 97, |
+ /* 20 */ 73, 89, 97, 90, 97, 97, 87, 97, 72, |
+}; |
+/********** End of lemon-generated parsing tables *****************************/ |
+ |
+/* The next table maps tokens (terminal symbols) into fallback tokens. |
+** If a construct like the following: |
+** |
+** %fallback ID X Y Z. |
+** |
+** appears in the grammar, then ID becomes a fallback token for X, Y, |
+** and Z. Whenever one of the tokens X, Y, or Z is input to the parser |
+** but it does not parse, the type of the token is changed to ID and |
+** the parse is retried before an error is thrown. |
+** |
+** This feature can be used, for example, to cause some keywords in a language |
+** to revert to identifiers if they keyword does not apply in the context where |
+** it appears. |
+*/ |
+#ifdef fts5YYFALLBACK |
+static const fts5YYCODETYPE fts5yyFallback[] = { |
+}; |
+#endif /* fts5YYFALLBACK */ |
+ |
+/* The following structure represents a single element of the |
+** parser's stack. Information stored includes: |
+** |
+** + The state number for the parser at this level of the stack. |
+** |
+** + The value of the token stored at this level of the stack. |
+** (In other words, the "major" token.) |
+** |
+** + The semantic value stored at this level of the stack. This is |
+** the information used by the action routines in the grammar. |
+** It is sometimes called the "minor" token. |
+** |
+** After the "shift" half of a SHIFTREDUCE action, the stateno field |
+** actually contains the reduce action for the second half of the |
+** SHIFTREDUCE. |
+*/ |
+struct fts5yyStackEntry { |
+ fts5YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ |
+ fts5YYCODETYPE major; /* The major token value. This is the code |
+ ** number for the token at this stack level */ |
+ fts5YYMINORTYPE minor; /* The user-supplied minor token value. This |
+ ** is the value of the token */ |
+}; |
+typedef struct fts5yyStackEntry fts5yyStackEntry; |
+ |
+/* The state of the parser is completely contained in an instance of |
+** the following structure */ |
+struct fts5yyParser { |
+ fts5yyStackEntry *fts5yytos; /* Pointer to top element of the stack */ |
+#ifdef fts5YYTRACKMAXSTACKDEPTH |
+ int fts5yyhwm; /* High-water mark of the stack */ |
+#endif |
+#ifndef fts5YYNOERRORRECOVERY |
+ int fts5yyerrcnt; /* Shifts left before out of the error */ |
+#endif |
+ sqlite3Fts5ParserARG_SDECL /* A place to hold %extra_argument */ |
+#if fts5YYSTACKDEPTH<=0 |
+ int fts5yystksz; /* Current side of the stack */ |
+ fts5yyStackEntry *fts5yystack; /* The parser's stack */ |
+ fts5yyStackEntry fts5yystk0; /* First stack entry */ |
+#else |
+ fts5yyStackEntry fts5yystack[fts5YYSTACKDEPTH]; /* The parser's stack */ |
+#endif |
+}; |
+typedef struct fts5yyParser fts5yyParser; |
+ |
+#ifndef NDEBUG |
+/* #include <stdio.h> */ |
+static FILE *fts5yyTraceFILE = 0; |
+static char *fts5yyTracePrompt = 0; |
+#endif /* NDEBUG */ |
+ |
+#ifndef NDEBUG |
+/* |
+** Turn parser tracing on by giving a stream to which to write the trace |
+** and a prompt to preface each trace message. Tracing is turned off |
+** by making either argument NULL |
+** |
+** Inputs: |
+** <ul> |
+** <li> A FILE* to which trace output should be written. |
+** If NULL, then tracing is turned off. |
+** <li> A prefix string written at the beginning of every |
+** line of trace output. If NULL, then tracing is |
+** turned off. |
+** </ul> |
+** |
+** Outputs: |
+** None. |
+*/ |
+static void sqlite3Fts5ParserTrace(FILE *TraceFILE, char *zTracePrompt){ |
+ fts5yyTraceFILE = TraceFILE; |
+ fts5yyTracePrompt = zTracePrompt; |
+ if( fts5yyTraceFILE==0 ) fts5yyTracePrompt = 0; |
+ else if( fts5yyTracePrompt==0 ) fts5yyTraceFILE = 0; |
+} |
+#endif /* NDEBUG */ |
+ |
+#ifndef NDEBUG |
+/* For tracing shifts, the names of all terminals and nonterminals |
+** are required. The following table supplies these names */ |
+static const char *const fts5yyTokenName[] = { |
+ "$", "OR", "AND", "NOT", |
+ "TERM", "COLON", "LP", "RP", |
+ "MINUS", "LCP", "RCP", "STRING", |
+ "COMMA", "PLUS", "STAR", "error", |
+ "input", "expr", "cnearset", "exprlist", |
+ "nearset", "colset", "colsetlist", "nearphrases", |
+ "phrase", "neardist_opt", "star_opt", |
+}; |
+#endif /* NDEBUG */ |
+ |
+#ifndef NDEBUG |
+/* For tracing reduce actions, the names of all rules are required. |
+*/ |
+static const char *const fts5yyRuleName[] = { |
+ /* 0 */ "input ::= expr", |
+ /* 1 */ "expr ::= expr AND expr", |
+ /* 2 */ "expr ::= expr OR expr", |
+ /* 3 */ "expr ::= expr NOT expr", |
+ /* 4 */ "expr ::= LP expr RP", |
+ /* 5 */ "expr ::= exprlist", |
+ /* 6 */ "exprlist ::= cnearset", |
+ /* 7 */ "exprlist ::= exprlist cnearset", |
+ /* 8 */ "cnearset ::= nearset", |
+ /* 9 */ "cnearset ::= colset COLON nearset", |
+ /* 10 */ "colset ::= MINUS LCP colsetlist RCP", |
+ /* 11 */ "colset ::= LCP colsetlist RCP", |
+ /* 12 */ "colset ::= STRING", |
+ /* 13 */ "colset ::= MINUS STRING", |
+ /* 14 */ "colsetlist ::= colsetlist STRING", |
+ /* 15 */ "colsetlist ::= STRING", |
+ /* 16 */ "nearset ::= phrase", |
+ /* 17 */ "nearset ::= STRING LP nearphrases neardist_opt RP", |
+ /* 18 */ "nearphrases ::= phrase", |
+ /* 19 */ "nearphrases ::= nearphrases phrase", |
+ /* 20 */ "neardist_opt ::=", |
+ /* 21 */ "neardist_opt ::= COMMA STRING", |
+ /* 22 */ "phrase ::= phrase PLUS STRING star_opt", |
+ /* 23 */ "phrase ::= STRING star_opt", |
+ /* 24 */ "star_opt ::= STAR", |
+ /* 25 */ "star_opt ::=", |
+}; |
+#endif /* NDEBUG */ |
+ |
+ |
+#if fts5YYSTACKDEPTH<=0 |
+/* |
+** Try to increase the size of the parser stack. Return the number |
+** of errors. Return 0 on success. |
+*/ |
+static int fts5yyGrowStack(fts5yyParser *p){ |
+ int newSize; |
+ int idx; |
+ fts5yyStackEntry *pNew; |
+ |
+ newSize = p->fts5yystksz*2 + 100; |
+ idx = p->fts5yytos ? (int)(p->fts5yytos - p->fts5yystack) : 0; |
+ if( p->fts5yystack==&p->fts5yystk0 ){ |
+ pNew = malloc(newSize*sizeof(pNew[0])); |
+ if( pNew ) pNew[0] = p->fts5yystk0; |
+ }else{ |
+ pNew = realloc(p->fts5yystack, newSize*sizeof(pNew[0])); |
+ } |
+ if( pNew ){ |
+ p->fts5yystack = pNew; |
+ p->fts5yytos = &p->fts5yystack[idx]; |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE ){ |
+ fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n", |
+ fts5yyTracePrompt, p->fts5yystksz, newSize); |
+ } |
+#endif |
+ p->fts5yystksz = newSize; |
+ } |
+ return pNew==0; |
+} |
+#endif |
+ |
+/* Datatype of the argument to the memory allocated passed as the |
+** second argument to sqlite3Fts5ParserAlloc() below. This can be changed by |
+** putting an appropriate #define in the %include section of the input |
+** grammar. |
+*/ |
+#ifndef fts5YYMALLOCARGTYPE |
+# define fts5YYMALLOCARGTYPE size_t |
+#endif |
+ |
+/* Initialize a new parser that has already been allocated. |
+*/ |
+static void sqlite3Fts5ParserInit(void *fts5yypParser){ |
+ fts5yyParser *pParser = (fts5yyParser*)fts5yypParser; |
+#ifdef fts5YYTRACKMAXSTACKDEPTH |
+ pParser->fts5yyhwm = 0; |
+#endif |
+#if fts5YYSTACKDEPTH<=0 |
+ pParser->fts5yytos = NULL; |
+ pParser->fts5yystack = NULL; |
+ pParser->fts5yystksz = 0; |
+ if( fts5yyGrowStack(pParser) ){ |
+ pParser->fts5yystack = &pParser->fts5yystk0; |
+ pParser->fts5yystksz = 1; |
+ } |
+#endif |
+#ifndef fts5YYNOERRORRECOVERY |
+ pParser->fts5yyerrcnt = -1; |
+#endif |
+ pParser->fts5yytos = pParser->fts5yystack; |
+ pParser->fts5yystack[0].stateno = 0; |
+ pParser->fts5yystack[0].major = 0; |
+} |
+ |
+#ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK |
+/* |
+** This function allocates a new parser. |
+** The only argument is a pointer to a function which works like |
+** malloc. |
+** |
+** Inputs: |
+** A pointer to the function used to allocate memory. |
+** |
+** Outputs: |
+** A pointer to a parser. This pointer is used in subsequent calls |
+** to sqlite3Fts5Parser and sqlite3Fts5ParserFree. |
+*/ |
+static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(fts5YYMALLOCARGTYPE)){ |
+ fts5yyParser *pParser; |
+ pParser = (fts5yyParser*)(*mallocProc)( (fts5YYMALLOCARGTYPE)sizeof(fts5yyParser) ); |
+ if( pParser ) sqlite3Fts5ParserInit(pParser); |
+ return pParser; |
+} |
+#endif /* sqlite3Fts5Parser_ENGINEALWAYSONSTACK */ |
+ |
+ |
+/* The following function deletes the "minor type" or semantic value |
+** associated with a symbol. The symbol can be either a terminal |
+** or nonterminal. "fts5yymajor" is the symbol code, and "fts5yypminor" is |
+** a pointer to the value to be deleted. The code used to do the |
+** deletions is derived from the %destructor and/or %token_destructor |
+** directives of the input grammar. |
+*/ |
+static void fts5yy_destructor( |
+ fts5yyParser *fts5yypParser, /* The parser */ |
+ fts5YYCODETYPE fts5yymajor, /* Type code for object to destroy */ |
+ fts5YYMINORTYPE *fts5yypminor /* The object to be destroyed */ |
+){ |
+ sqlite3Fts5ParserARG_FETCH; |
+ switch( fts5yymajor ){ |
+ /* Here is inserted the actions which take place when a |
+ ** terminal or non-terminal is destroyed. This can happen |
+ ** when the symbol is popped from the stack during a |
+ ** reduce or during error processing or when a parser is |
+ ** being destroyed before it is finished parsing. |
+ ** |
+ ** Note: during a reduce, the only symbols destroyed are those |
+ ** which appear on the RHS of the rule, but which are *not* used |
+ ** inside the C code. |
+ */ |
+/********* Begin destructor definitions ***************************************/ |
+ case 16: /* input */ |
+{ |
+ (void)pParse; |
+} |
+ break; |
+ case 17: /* expr */ |
+ case 18: /* cnearset */ |
+ case 19: /* exprlist */ |
+{ |
+ sqlite3Fts5ParseNodeFree((fts5yypminor->fts5yy24)); |
+} |
+ break; |
+ case 20: /* nearset */ |
+ case 23: /* nearphrases */ |
+{ |
+ sqlite3Fts5ParseNearsetFree((fts5yypminor->fts5yy46)); |
+} |
+ break; |
+ case 21: /* colset */ |
+ case 22: /* colsetlist */ |
+{ |
+ sqlite3_free((fts5yypminor->fts5yy11)); |
+} |
+ break; |
+ case 24: /* phrase */ |
+{ |
+ sqlite3Fts5ParsePhraseFree((fts5yypminor->fts5yy53)); |
+} |
+ break; |
+/********* End destructor definitions *****************************************/ |
+ default: break; /* If no destructor action specified: do nothing */ |
+ } |
+} |
+ |
+/* |
+** Pop the parser's stack once. |
+** |
+** If there is a destructor routine associated with the token which |
+** is popped from the stack, then call it. |
+*/ |
+static void fts5yy_pop_parser_stack(fts5yyParser *pParser){ |
+ fts5yyStackEntry *fts5yytos; |
+ assert( pParser->fts5yytos!=0 ); |
+ assert( pParser->fts5yytos > pParser->fts5yystack ); |
+ fts5yytos = pParser->fts5yytos--; |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE ){ |
+ fprintf(fts5yyTraceFILE,"%sPopping %s\n", |
+ fts5yyTracePrompt, |
+ fts5yyTokenName[fts5yytos->major]); |
+ } |
+#endif |
+ fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor); |
+} |
+ |
+/* |
+** Clear all secondary memory allocations from the parser |
+*/ |
+static void sqlite3Fts5ParserFinalize(void *p){ |
+ fts5yyParser *pParser = (fts5yyParser*)p; |
+ while( pParser->fts5yytos>pParser->fts5yystack ) fts5yy_pop_parser_stack(pParser); |
+#if fts5YYSTACKDEPTH<=0 |
+ if( pParser->fts5yystack!=&pParser->fts5yystk0 ) free(pParser->fts5yystack); |
+#endif |
+} |
+ |
+#ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK |
+/* |
+** Deallocate and destroy a parser. Destructors are called for |
+** all stack elements before shutting the parser down. |
+** |
+** If the fts5YYPARSEFREENEVERNULL macro exists (for example because it |
+** is defined in a %include section of the input grammar) then it is |
+** assumed that the input pointer is never NULL. |
+*/ |
+static void sqlite3Fts5ParserFree( |
+ void *p, /* The parser to be deleted */ |
+ void (*freeProc)(void*) /* Function used to reclaim memory */ |
+){ |
+#ifndef fts5YYPARSEFREENEVERNULL |
+ if( p==0 ) return; |
+#endif |
+ sqlite3Fts5ParserFinalize(p); |
+ (*freeProc)(p); |
+} |
+#endif /* sqlite3Fts5Parser_ENGINEALWAYSONSTACK */ |
+ |
+/* |
+** Return the peak depth of the stack for a parser. |
+*/ |
+#ifdef fts5YYTRACKMAXSTACKDEPTH |
+static int sqlite3Fts5ParserStackPeak(void *p){ |
+ fts5yyParser *pParser = (fts5yyParser*)p; |
+ return pParser->fts5yyhwm; |
+} |
+#endif |
+ |
+/* |
+** Find the appropriate action for a parser given the terminal |
+** look-ahead token iLookAhead. |
+*/ |
+static unsigned int fts5yy_find_shift_action( |
+ fts5yyParser *pParser, /* The parser */ |
+ fts5YYCODETYPE iLookAhead /* The look-ahead token */ |
+){ |
+ int i; |
+ int stateno = pParser->fts5yytos->stateno; |
+ |
+ if( stateno>=fts5YY_MIN_REDUCE ) return stateno; |
+ assert( stateno <= fts5YY_SHIFT_COUNT ); |
+ do{ |
+ i = fts5yy_shift_ofst[stateno]; |
+ assert( iLookAhead!=fts5YYNOCODE ); |
+ i += iLookAhead; |
+ if( i<0 || i>=fts5YY_ACTTAB_COUNT || fts5yy_lookahead[i]!=iLookAhead ){ |
+#ifdef fts5YYFALLBACK |
+ fts5YYCODETYPE iFallback; /* Fallback token */ |
+ if( iLookAhead<sizeof(fts5yyFallback)/sizeof(fts5yyFallback[0]) |
+ && (iFallback = fts5yyFallback[iLookAhead])!=0 ){ |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE ){ |
+ fprintf(fts5yyTraceFILE, "%sFALLBACK %s => %s\n", |
+ fts5yyTracePrompt, fts5yyTokenName[iLookAhead], fts5yyTokenName[iFallback]); |
+ } |
+#endif |
+ assert( fts5yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ |
+ iLookAhead = iFallback; |
+ continue; |
+ } |
+#endif |
+#ifdef fts5YYWILDCARD |
+ { |
+ int j = i - iLookAhead + fts5YYWILDCARD; |
+ if( |
+#if fts5YY_SHIFT_MIN+fts5YYWILDCARD<0 |
+ j>=0 && |
+#endif |
+#if fts5YY_SHIFT_MAX+fts5YYWILDCARD>=fts5YY_ACTTAB_COUNT |
+ j<fts5YY_ACTTAB_COUNT && |
+#endif |
+ fts5yy_lookahead[j]==fts5YYWILDCARD && iLookAhead>0 |
+ ){ |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE ){ |
+ fprintf(fts5yyTraceFILE, "%sWILDCARD %s => %s\n", |
+ fts5yyTracePrompt, fts5yyTokenName[iLookAhead], |
+ fts5yyTokenName[fts5YYWILDCARD]); |
+ } |
+#endif /* NDEBUG */ |
+ return fts5yy_action[j]; |
+ } |
+ } |
+#endif /* fts5YYWILDCARD */ |
+ return fts5yy_default[stateno]; |
+ }else{ |
+ return fts5yy_action[i]; |
+ } |
+ }while(1); |
+} |
+ |
+/* |
+** Find the appropriate action for a parser given the non-terminal |
+** look-ahead token iLookAhead. |
+*/ |
+static int fts5yy_find_reduce_action( |
+ int stateno, /* Current state number */ |
+ fts5YYCODETYPE iLookAhead /* The look-ahead token */ |
+){ |
+ int i; |
+#ifdef fts5YYERRORSYMBOL |
+ if( stateno>fts5YY_REDUCE_COUNT ){ |
+ return fts5yy_default[stateno]; |
+ } |
+#else |
+ assert( stateno<=fts5YY_REDUCE_COUNT ); |
+#endif |
+ i = fts5yy_reduce_ofst[stateno]; |
+ assert( i!=fts5YY_REDUCE_USE_DFLT ); |
+ assert( iLookAhead!=fts5YYNOCODE ); |
+ i += iLookAhead; |
+#ifdef fts5YYERRORSYMBOL |
+ if( i<0 || i>=fts5YY_ACTTAB_COUNT || fts5yy_lookahead[i]!=iLookAhead ){ |
+ return fts5yy_default[stateno]; |
+ } |
+#else |
+ assert( i>=0 && i<fts5YY_ACTTAB_COUNT ); |
+ assert( fts5yy_lookahead[i]==iLookAhead ); |
+#endif |
+ return fts5yy_action[i]; |
+} |
+ |
+/* |
+** The following routine is called if the stack overflows. |
+*/ |
+static void fts5yyStackOverflow(fts5yyParser *fts5yypParser){ |
+ sqlite3Fts5ParserARG_FETCH; |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE ){ |
+ fprintf(fts5yyTraceFILE,"%sStack Overflow!\n",fts5yyTracePrompt); |
+ } |
+#endif |
+ while( fts5yypParser->fts5yytos>fts5yypParser->fts5yystack ) fts5yy_pop_parser_stack(fts5yypParser); |
+ /* Here code is inserted which will execute if the parser |
+ ** stack every overflows */ |
+/******** Begin %stack_overflow code ******************************************/ |
+ |
+ sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow"); |
+/******** End %stack_overflow code ********************************************/ |
+ sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument var */ |
+} |
+ |
+/* |
+** Print tracing information for a SHIFT action |
+*/ |
+#ifndef NDEBUG |
+static void fts5yyTraceShift(fts5yyParser *fts5yypParser, int fts5yyNewState){ |
+ if( fts5yyTraceFILE ){ |
+ if( fts5yyNewState<fts5YYNSTATE ){ |
+ fprintf(fts5yyTraceFILE,"%sShift '%s', go to state %d\n", |
+ fts5yyTracePrompt,fts5yyTokenName[fts5yypParser->fts5yytos->major], |
+ fts5yyNewState); |
+ }else{ |
+ fprintf(fts5yyTraceFILE,"%sShift '%s'\n", |
+ fts5yyTracePrompt,fts5yyTokenName[fts5yypParser->fts5yytos->major]); |
+ } |
+ } |
+} |
+#else |
+# define fts5yyTraceShift(X,Y) |
+#endif |
+ |
+/* |
+** Perform a shift action. |
+*/ |
+static void fts5yy_shift( |
+ fts5yyParser *fts5yypParser, /* The parser to be shifted */ |
+ int fts5yyNewState, /* The new state to shift in */ |
+ int fts5yyMajor, /* The major token to shift in */ |
+ sqlite3Fts5ParserFTS5TOKENTYPE fts5yyMinor /* The minor token to shift in */ |
+){ |
+ fts5yyStackEntry *fts5yytos; |
+ fts5yypParser->fts5yytos++; |
+#ifdef fts5YYTRACKMAXSTACKDEPTH |
+ if( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)>fts5yypParser->fts5yyhwm ){ |
+ fts5yypParser->fts5yyhwm++; |
+ assert( fts5yypParser->fts5yyhwm == (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack) ); |
+ } |
+#endif |
+#if fts5YYSTACKDEPTH>0 |
+ if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5YYSTACKDEPTH] ){ |
+ fts5yypParser->fts5yytos--; |
+ fts5yyStackOverflow(fts5yypParser); |
+ return; |
+ } |
+#else |
+ if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz] ){ |
+ if( fts5yyGrowStack(fts5yypParser) ){ |
+ fts5yypParser->fts5yytos--; |
+ fts5yyStackOverflow(fts5yypParser); |
+ return; |
+ } |
+ } |
+#endif |
+ if( fts5yyNewState > fts5YY_MAX_SHIFT ){ |
+ fts5yyNewState += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; |
+ } |
+ fts5yytos = fts5yypParser->fts5yytos; |
+ fts5yytos->stateno = (fts5YYACTIONTYPE)fts5yyNewState; |
+ fts5yytos->major = (fts5YYCODETYPE)fts5yyMajor; |
+ fts5yytos->minor.fts5yy0 = fts5yyMinor; |
+ fts5yyTraceShift(fts5yypParser, fts5yyNewState); |
+} |
+ |
+/* The following table contains information about every rule that |
+** is used during the reduce. |
+*/ |
+static const struct { |
+ fts5YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ |
+ unsigned char nrhs; /* Number of right-hand side symbols in the rule */ |
+} fts5yyRuleInfo[] = { |
+ { 16, 1 }, |
+ { 17, 3 }, |
+ { 17, 3 }, |
+ { 17, 3 }, |
+ { 17, 3 }, |
+ { 17, 1 }, |
+ { 19, 1 }, |
+ { 19, 2 }, |
+ { 18, 1 }, |
+ { 18, 3 }, |
+ { 21, 4 }, |
+ { 21, 3 }, |
+ { 21, 1 }, |
+ { 21, 2 }, |
+ { 22, 2 }, |
+ { 22, 1 }, |
+ { 20, 1 }, |
+ { 20, 5 }, |
+ { 23, 1 }, |
+ { 23, 2 }, |
+ { 25, 0 }, |
+ { 25, 2 }, |
+ { 24, 4 }, |
+ { 24, 2 }, |
+ { 26, 1 }, |
+ { 26, 0 }, |
+}; |
+ |
+static void fts5yy_accept(fts5yyParser*); /* Forward Declaration */ |
+ |
+/* |
+** Perform a reduce action and the shift that must immediately |
+** follow the reduce. |
+*/ |
+static void fts5yy_reduce( |
+ fts5yyParser *fts5yypParser, /* The parser */ |
+ unsigned int fts5yyruleno /* Number of the rule by which to reduce */ |
+){ |
+ int fts5yygoto; /* The next state */ |
+ int fts5yyact; /* The next action */ |
+ fts5yyStackEntry *fts5yymsp; /* The top of the parser's stack */ |
+ int fts5yysize; /* Amount to pop the stack */ |
+ sqlite3Fts5ParserARG_FETCH; |
+ fts5yymsp = fts5yypParser->fts5yytos; |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE && fts5yyruleno<(int)(sizeof(fts5yyRuleName)/sizeof(fts5yyRuleName[0])) ){ |
+ fts5yysize = fts5yyRuleInfo[fts5yyruleno].nrhs; |
+ fprintf(fts5yyTraceFILE, "%sReduce [%s], go to state %d.\n", fts5yyTracePrompt, |
+ fts5yyRuleName[fts5yyruleno], fts5yymsp[-fts5yysize].stateno); |
+ } |
+#endif /* NDEBUG */ |
+ |
+ /* Check that the stack is large enough to grow by a single entry |
+ ** if the RHS of the rule is empty. This ensures that there is room |
+ ** enough on the stack to push the LHS value */ |
+ if( fts5yyRuleInfo[fts5yyruleno].nrhs==0 ){ |
+#ifdef fts5YYTRACKMAXSTACKDEPTH |
+ if( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)>fts5yypParser->fts5yyhwm ){ |
+ fts5yypParser->fts5yyhwm++; |
+ assert( fts5yypParser->fts5yyhwm == (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)); |
+ } |
+#endif |
+#if fts5YYSTACKDEPTH>0 |
+ if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5YYSTACKDEPTH-1] ){ |
+ fts5yyStackOverflow(fts5yypParser); |
+ return; |
+ } |
+#else |
+ if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz-1] ){ |
+ if( fts5yyGrowStack(fts5yypParser) ){ |
+ fts5yyStackOverflow(fts5yypParser); |
+ return; |
+ } |
+ fts5yymsp = fts5yypParser->fts5yytos; |
+ } |
+#endif |
+ } |
+ |
+ switch( fts5yyruleno ){ |
+ /* Beginning here are the reduction cases. A typical example |
+ ** follows: |
+ ** case 0: |
+ ** #line <lineno> <grammarfile> |
+ ** { ... } // User supplied code |
+ ** #line <lineno> <thisfile> |
+ ** break; |
+ */ |
+/********** Begin reduce actions **********************************************/ |
+ fts5YYMINORTYPE fts5yylhsminor; |
+ case 0: /* input ::= expr */ |
+{ sqlite3Fts5ParseFinished(pParse, fts5yymsp[0].minor.fts5yy24); } |
+ break; |
+ case 1: /* expr ::= expr AND expr */ |
+{ |
+ fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_AND, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0); |
+} |
+ fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; |
+ break; |
+ case 2: /* expr ::= expr OR expr */ |
+{ |
+ fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_OR, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0); |
+} |
+ fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; |
+ break; |
+ case 3: /* expr ::= expr NOT expr */ |
+{ |
+ fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_NOT, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0); |
+} |
+ fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; |
+ break; |
+ case 4: /* expr ::= LP expr RP */ |
+{fts5yymsp[-2].minor.fts5yy24 = fts5yymsp[-1].minor.fts5yy24;} |
+ break; |
+ case 5: /* expr ::= exprlist */ |
+ case 6: /* exprlist ::= cnearset */ fts5yytestcase(fts5yyruleno==6); |
+{fts5yylhsminor.fts5yy24 = fts5yymsp[0].minor.fts5yy24;} |
+ fts5yymsp[0].minor.fts5yy24 = fts5yylhsminor.fts5yy24; |
+ break; |
+ case 7: /* exprlist ::= exprlist cnearset */ |
+{ |
+ fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseImplicitAnd(pParse, fts5yymsp[-1].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24); |
+} |
+ fts5yymsp[-1].minor.fts5yy24 = fts5yylhsminor.fts5yy24; |
+ break; |
+ case 8: /* cnearset ::= nearset */ |
+{ |
+ fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy46); |
+} |
+ fts5yymsp[0].minor.fts5yy24 = fts5yylhsminor.fts5yy24; |
+ break; |
+ case 9: /* cnearset ::= colset COLON nearset */ |
+{ |
+ sqlite3Fts5ParseSetColset(pParse, fts5yymsp[0].minor.fts5yy46, fts5yymsp[-2].minor.fts5yy11); |
+ fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy46); |
+} |
+ fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; |
+ break; |
+ case 10: /* colset ::= MINUS LCP colsetlist RCP */ |
+{ |
+ fts5yymsp[-3].minor.fts5yy11 = sqlite3Fts5ParseColsetInvert(pParse, fts5yymsp[-1].minor.fts5yy11); |
+} |
+ break; |
+ case 11: /* colset ::= LCP colsetlist RCP */ |
+{ fts5yymsp[-2].minor.fts5yy11 = fts5yymsp[-1].minor.fts5yy11; } |
+ break; |
+ case 12: /* colset ::= STRING */ |
+{ |
+ fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); |
+} |
+ fts5yymsp[0].minor.fts5yy11 = fts5yylhsminor.fts5yy11; |
+ break; |
+ case 13: /* colset ::= MINUS STRING */ |
+{ |
+ fts5yymsp[-1].minor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); |
+ fts5yymsp[-1].minor.fts5yy11 = sqlite3Fts5ParseColsetInvert(pParse, fts5yymsp[-1].minor.fts5yy11); |
+} |
+ break; |
+ case 14: /* colsetlist ::= colsetlist STRING */ |
+{ |
+ fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, fts5yymsp[-1].minor.fts5yy11, &fts5yymsp[0].minor.fts5yy0); } |
+ fts5yymsp[-1].minor.fts5yy11 = fts5yylhsminor.fts5yy11; |
+ break; |
+ case 15: /* colsetlist ::= STRING */ |
+{ |
+ fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); |
+} |
+ fts5yymsp[0].minor.fts5yy11 = fts5yylhsminor.fts5yy11; |
+ break; |
+ case 16: /* nearset ::= phrase */ |
+{ fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53); } |
+ fts5yymsp[0].minor.fts5yy46 = fts5yylhsminor.fts5yy46; |
+ break; |
+ case 17: /* nearset ::= STRING LP nearphrases neardist_opt RP */ |
+{ |
+ sqlite3Fts5ParseNear(pParse, &fts5yymsp[-4].minor.fts5yy0); |
+ sqlite3Fts5ParseSetDistance(pParse, fts5yymsp[-2].minor.fts5yy46, &fts5yymsp[-1].minor.fts5yy0); |
+ fts5yylhsminor.fts5yy46 = fts5yymsp[-2].minor.fts5yy46; |
+} |
+ fts5yymsp[-4].minor.fts5yy46 = fts5yylhsminor.fts5yy46; |
+ break; |
+ case 18: /* nearphrases ::= phrase */ |
+{ |
+ fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53); |
+} |
+ fts5yymsp[0].minor.fts5yy46 = fts5yylhsminor.fts5yy46; |
+ break; |
+ case 19: /* nearphrases ::= nearphrases phrase */ |
+{ |
+ fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, fts5yymsp[-1].minor.fts5yy46, fts5yymsp[0].minor.fts5yy53); |
+} |
+ fts5yymsp[-1].minor.fts5yy46 = fts5yylhsminor.fts5yy46; |
+ break; |
+ case 20: /* neardist_opt ::= */ |
+{ fts5yymsp[1].minor.fts5yy0.p = 0; fts5yymsp[1].minor.fts5yy0.n = 0; } |
+ break; |
+ case 21: /* neardist_opt ::= COMMA STRING */ |
+{ fts5yymsp[-1].minor.fts5yy0 = fts5yymsp[0].minor.fts5yy0; } |
+ break; |
+ case 22: /* phrase ::= phrase PLUS STRING star_opt */ |
+{ |
+ fts5yylhsminor.fts5yy53 = sqlite3Fts5ParseTerm(pParse, fts5yymsp[-3].minor.fts5yy53, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy4); |
+} |
+ fts5yymsp[-3].minor.fts5yy53 = fts5yylhsminor.fts5yy53; |
+ break; |
+ case 23: /* phrase ::= STRING star_opt */ |
+{ |
+ fts5yylhsminor.fts5yy53 = sqlite3Fts5ParseTerm(pParse, 0, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy4); |
+} |
+ fts5yymsp[-1].minor.fts5yy53 = fts5yylhsminor.fts5yy53; |
+ break; |
+ case 24: /* star_opt ::= STAR */ |
+{ fts5yymsp[0].minor.fts5yy4 = 1; } |
+ break; |
+ case 25: /* star_opt ::= */ |
+{ fts5yymsp[1].minor.fts5yy4 = 0; } |
+ break; |
+ default: |
+ break; |
+/********** End reduce actions ************************************************/ |
+ }; |
+ assert( fts5yyruleno<sizeof(fts5yyRuleInfo)/sizeof(fts5yyRuleInfo[0]) ); |
+ fts5yygoto = fts5yyRuleInfo[fts5yyruleno].lhs; |
+ fts5yysize = fts5yyRuleInfo[fts5yyruleno].nrhs; |
+ fts5yyact = fts5yy_find_reduce_action(fts5yymsp[-fts5yysize].stateno,(fts5YYCODETYPE)fts5yygoto); |
+ if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ |
+ if( fts5yyact>fts5YY_MAX_SHIFT ){ |
+ fts5yyact += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; |
+ } |
+ fts5yymsp -= fts5yysize-1; |
+ fts5yypParser->fts5yytos = fts5yymsp; |
+ fts5yymsp->stateno = (fts5YYACTIONTYPE)fts5yyact; |
+ fts5yymsp->major = (fts5YYCODETYPE)fts5yygoto; |
+ fts5yyTraceShift(fts5yypParser, fts5yyact); |
+ }else{ |
+ assert( fts5yyact == fts5YY_ACCEPT_ACTION ); |
+ fts5yypParser->fts5yytos -= fts5yysize; |
+ fts5yy_accept(fts5yypParser); |
+ } |
+} |
+ |
+/* |
+** The following code executes when the parse fails |
+*/ |
+#ifndef fts5YYNOERRORRECOVERY |
+static void fts5yy_parse_failed( |
+ fts5yyParser *fts5yypParser /* The parser */ |
+){ |
+ sqlite3Fts5ParserARG_FETCH; |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE ){ |
+ fprintf(fts5yyTraceFILE,"%sFail!\n",fts5yyTracePrompt); |
+ } |
+#endif |
+ while( fts5yypParser->fts5yytos>fts5yypParser->fts5yystack ) fts5yy_pop_parser_stack(fts5yypParser); |
+ /* Here code is inserted which will be executed whenever the |
+ ** parser fails */ |
+/************ Begin %parse_failure code ***************************************/ |
+/************ End %parse_failure code *****************************************/ |
+ sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ |
+} |
+#endif /* fts5YYNOERRORRECOVERY */ |
+ |
+/* |
+** The following code executes when a syntax error first occurs. |
+*/ |
+static void fts5yy_syntax_error( |
+ fts5yyParser *fts5yypParser, /* The parser */ |
+ int fts5yymajor, /* The major type of the error token */ |
+ sqlite3Fts5ParserFTS5TOKENTYPE fts5yyminor /* The minor type of the error token */ |
+){ |
+ sqlite3Fts5ParserARG_FETCH; |
+#define FTS5TOKEN fts5yyminor |
+/************ Begin %syntax_error code ****************************************/ |
+ |
+ UNUSED_PARAM(fts5yymajor); /* Silence a compiler warning */ |
+ sqlite3Fts5ParseError( |
+ pParse, "fts5: syntax error near \"%.*s\"",FTS5TOKEN.n,FTS5TOKEN.p |
+ ); |
+/************ End %syntax_error code ******************************************/ |
+ sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ |
+} |
+ |
+/* |
+** The following is executed when the parser accepts |
+*/ |
+static void fts5yy_accept( |
+ fts5yyParser *fts5yypParser /* The parser */ |
+){ |
+ sqlite3Fts5ParserARG_FETCH; |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE ){ |
+ fprintf(fts5yyTraceFILE,"%sAccept!\n",fts5yyTracePrompt); |
+ } |
+#endif |
+#ifndef fts5YYNOERRORRECOVERY |
+ fts5yypParser->fts5yyerrcnt = -1; |
+#endif |
+ assert( fts5yypParser->fts5yytos==fts5yypParser->fts5yystack ); |
+ /* Here code is inserted which will be executed whenever the |
+ ** parser accepts */ |
+/*********** Begin %parse_accept code *****************************************/ |
+/*********** End %parse_accept code *******************************************/ |
+ sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ |
+} |
+ |
+/* The main parser program. |
+** The first argument is a pointer to a structure obtained from |
+** "sqlite3Fts5ParserAlloc" which describes the current state of the parser. |
+** The second argument is the major token number. The third is |
+** the minor token. The fourth optional argument is whatever the |
+** user wants (and specified in the grammar) and is available for |
+** use by the action routines. |
+** |
+** Inputs: |
+** <ul> |
+** <li> A pointer to the parser (an opaque structure.) |
+** <li> The major token number. |
+** <li> The minor token number. |
+** <li> An option argument of a grammar-specified type. |
+** </ul> |
+** |
+** Outputs: |
+** None. |
+*/ |
+static void sqlite3Fts5Parser( |
+ void *fts5yyp, /* The parser */ |
+ int fts5yymajor, /* The major token code number */ |
+ sqlite3Fts5ParserFTS5TOKENTYPE fts5yyminor /* The value for the token */ |
+ sqlite3Fts5ParserARG_PDECL /* Optional %extra_argument parameter */ |
+){ |
+ fts5YYMINORTYPE fts5yyminorunion; |
+ unsigned int fts5yyact; /* The parser action. */ |
+#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY) |
+ int fts5yyendofinput; /* True if we are at the end of input */ |
+#endif |
+#ifdef fts5YYERRORSYMBOL |
+ int fts5yyerrorhit = 0; /* True if fts5yymajor has invoked an error */ |
+#endif |
+ fts5yyParser *fts5yypParser; /* The parser */ |
+ |
+ fts5yypParser = (fts5yyParser*)fts5yyp; |
+ assert( fts5yypParser->fts5yytos!=0 ); |
+#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY) |
+ fts5yyendofinput = (fts5yymajor==0); |
+#endif |
+ sqlite3Fts5ParserARG_STORE; |
+ |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE ){ |
+ fprintf(fts5yyTraceFILE,"%sInput '%s'\n",fts5yyTracePrompt,fts5yyTokenName[fts5yymajor]); |
+ } |
+#endif |
+ |
+ do{ |
+ fts5yyact = fts5yy_find_shift_action(fts5yypParser,(fts5YYCODETYPE)fts5yymajor); |
+ if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ |
+ fts5yy_shift(fts5yypParser,fts5yyact,fts5yymajor,fts5yyminor); |
+#ifndef fts5YYNOERRORRECOVERY |
+ fts5yypParser->fts5yyerrcnt--; |
+#endif |
+ fts5yymajor = fts5YYNOCODE; |
+ }else if( fts5yyact <= fts5YY_MAX_REDUCE ){ |
+ fts5yy_reduce(fts5yypParser,fts5yyact-fts5YY_MIN_REDUCE); |
+ }else{ |
+ assert( fts5yyact == fts5YY_ERROR_ACTION ); |
+ fts5yyminorunion.fts5yy0 = fts5yyminor; |
+#ifdef fts5YYERRORSYMBOL |
+ int fts5yymx; |
+#endif |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE ){ |
+ fprintf(fts5yyTraceFILE,"%sSyntax Error!\n",fts5yyTracePrompt); |
+ } |
+#endif |
+#ifdef fts5YYERRORSYMBOL |
+ /* A syntax error has occurred. |
+ ** The response to an error depends upon whether or not the |
+ ** grammar defines an error token "ERROR". |
+ ** |
+ ** This is what we do if the grammar does define ERROR: |
+ ** |
+ ** * Call the %syntax_error function. |
+ ** |
+ ** * Begin popping the stack until we enter a state where |
+ ** it is legal to shift the error symbol, then shift |
+ ** the error symbol. |
+ ** |
+ ** * Set the error count to three. |
+ ** |
+ ** * Begin accepting and shifting new tokens. No new error |
+ ** processing will occur until three tokens have been |
+ ** shifted successfully. |
+ ** |
+ */ |
+ if( fts5yypParser->fts5yyerrcnt<0 ){ |
+ fts5yy_syntax_error(fts5yypParser,fts5yymajor,fts5yyminor); |
+ } |
+ fts5yymx = fts5yypParser->fts5yytos->major; |
+ if( fts5yymx==fts5YYERRORSYMBOL || fts5yyerrorhit ){ |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE ){ |
+ fprintf(fts5yyTraceFILE,"%sDiscard input token %s\n", |
+ fts5yyTracePrompt,fts5yyTokenName[fts5yymajor]); |
+ } |
+#endif |
+ fts5yy_destructor(fts5yypParser, (fts5YYCODETYPE)fts5yymajor, &fts5yyminorunion); |
+ fts5yymajor = fts5YYNOCODE; |
+ }else{ |
+ while( fts5yypParser->fts5yytos >= fts5yypParser->fts5yystack |
+ && fts5yymx != fts5YYERRORSYMBOL |
+ && (fts5yyact = fts5yy_find_reduce_action( |
+ fts5yypParser->fts5yytos->stateno, |
+ fts5YYERRORSYMBOL)) >= fts5YY_MIN_REDUCE |
+ ){ |
+ fts5yy_pop_parser_stack(fts5yypParser); |
+ } |
+ if( fts5yypParser->fts5yytos < fts5yypParser->fts5yystack || fts5yymajor==0 ){ |
+ fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); |
+ fts5yy_parse_failed(fts5yypParser); |
+#ifndef fts5YYNOERRORRECOVERY |
+ fts5yypParser->fts5yyerrcnt = -1; |
+#endif |
+ fts5yymajor = fts5YYNOCODE; |
+ }else if( fts5yymx!=fts5YYERRORSYMBOL ){ |
+ fts5yy_shift(fts5yypParser,fts5yyact,fts5YYERRORSYMBOL,fts5yyminor); |
+ } |
+ } |
+ fts5yypParser->fts5yyerrcnt = 3; |
+ fts5yyerrorhit = 1; |
+#elif defined(fts5YYNOERRORRECOVERY) |
+ /* If the fts5YYNOERRORRECOVERY macro is defined, then do not attempt to |
+ ** do any kind of error recovery. Instead, simply invoke the syntax |
+ ** error routine and continue going as if nothing had happened. |
+ ** |
+ ** Applications can set this macro (for example inside %include) if |
+ ** they intend to abandon the parse upon the first syntax error seen. |
+ */ |
+ fts5yy_syntax_error(fts5yypParser,fts5yymajor, fts5yyminor); |
+ fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); |
+ fts5yymajor = fts5YYNOCODE; |
+ |
+#else /* fts5YYERRORSYMBOL is not defined */ |
+ /* This is what we do if the grammar does not define ERROR: |
+ ** |
+ ** * Report an error message, and throw away the input token. |
+ ** |
+ ** * If the input token is $, then fail the parse. |
+ ** |
+ ** As before, subsequent error messages are suppressed until |
+ ** three input tokens have been successfully shifted. |
+ */ |
+ if( fts5yypParser->fts5yyerrcnt<=0 ){ |
+ fts5yy_syntax_error(fts5yypParser,fts5yymajor, fts5yyminor); |
+ } |
+ fts5yypParser->fts5yyerrcnt = 3; |
+ fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); |
+ if( fts5yyendofinput ){ |
+ fts5yy_parse_failed(fts5yypParser); |
+#ifndef fts5YYNOERRORRECOVERY |
+ fts5yypParser->fts5yyerrcnt = -1; |
+#endif |
+ } |
+ fts5yymajor = fts5YYNOCODE; |
+#endif |
+ } |
+ }while( fts5yymajor!=fts5YYNOCODE && fts5yypParser->fts5yytos>fts5yypParser->fts5yystack ); |
+#ifndef NDEBUG |
+ if( fts5yyTraceFILE ){ |
+ fts5yyStackEntry *i; |
+ char cDiv = '['; |
+ fprintf(fts5yyTraceFILE,"%sReturn. Stack=",fts5yyTracePrompt); |
+ for(i=&fts5yypParser->fts5yystack[1]; i<=fts5yypParser->fts5yytos; i++){ |
+ fprintf(fts5yyTraceFILE,"%c%s", cDiv, fts5yyTokenName[i->major]); |
+ cDiv = ' '; |
+ } |
+ fprintf(fts5yyTraceFILE,"]\n"); |
+ } |
+#endif |
+ return; |
+} |
+ |
+/* |
+** 2014 May 31 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+*/ |
+ |
+ |
+/* #include "fts5Int.h" */ |
+#include <math.h> /* amalgamator: keep */ |
+ |
+/* |
+** Object used to iterate through all "coalesced phrase instances" in |
+** a single column of the current row. If the phrase instances in the |
+** column being considered do not overlap, this object simply iterates |
+** through them. Or, if they do overlap (share one or more tokens in |
+** common), each set of overlapping instances is treated as a single |
+** match. See documentation for the highlight() auxiliary function for |
+** details. |
+** |
+** Usage is: |
+** |
+** for(rc = fts5CInstIterNext(pApi, pFts, iCol, &iter); |
+** (rc==SQLITE_OK && 0==fts5CInstIterEof(&iter); |
+** rc = fts5CInstIterNext(&iter) |
+** ){ |
+** printf("instance starts at %d, ends at %d\n", iter.iStart, iter.iEnd); |
+** } |
+** |
+*/ |
+typedef struct CInstIter CInstIter; |
+struct CInstIter { |
+ const Fts5ExtensionApi *pApi; /* API offered by current FTS version */ |
+ Fts5Context *pFts; /* First arg to pass to pApi functions */ |
+ int iCol; /* Column to search */ |
+ int iInst; /* Next phrase instance index */ |
+ int nInst; /* Total number of phrase instances */ |
+ |
+ /* Output variables */ |
+ int iStart; /* First token in coalesced phrase instance */ |
+ int iEnd; /* Last token in coalesced phrase instance */ |
+}; |
+ |
+/* |
+** Advance the iterator to the next coalesced phrase instance. Return |
+** an SQLite error code if an error occurs, or SQLITE_OK otherwise. |
+*/ |
+static int fts5CInstIterNext(CInstIter *pIter){ |
+ int rc = SQLITE_OK; |
+ pIter->iStart = -1; |
+ pIter->iEnd = -1; |
+ |
+ while( rc==SQLITE_OK && pIter->iInst<pIter->nInst ){ |
+ int ip; int ic; int io; |
+ rc = pIter->pApi->xInst(pIter->pFts, pIter->iInst, &ip, &ic, &io); |
+ if( rc==SQLITE_OK ){ |
+ if( ic==pIter->iCol ){ |
+ int iEnd = io - 1 + pIter->pApi->xPhraseSize(pIter->pFts, ip); |
+ if( pIter->iStart<0 ){ |
+ pIter->iStart = io; |
+ pIter->iEnd = iEnd; |
+ }else if( io<=pIter->iEnd ){ |
+ if( iEnd>pIter->iEnd ) pIter->iEnd = iEnd; |
+ }else{ |
+ break; |
+ } |
+ } |
+ pIter->iInst++; |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Initialize the iterator object indicated by the final parameter to |
+** iterate through coalesced phrase instances in column iCol. |
+*/ |
+static int fts5CInstIterInit( |
+ const Fts5ExtensionApi *pApi, |
+ Fts5Context *pFts, |
+ int iCol, |
+ CInstIter *pIter |
+){ |
+ int rc; |
+ |
+ memset(pIter, 0, sizeof(CInstIter)); |
+ pIter->pApi = pApi; |
+ pIter->pFts = pFts; |
+ pIter->iCol = iCol; |
+ rc = pApi->xInstCount(pFts, &pIter->nInst); |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5CInstIterNext(pIter); |
+ } |
+ |
+ return rc; |
+} |
+ |
+ |
+ |
+/************************************************************************* |
+** Start of highlight() implementation. |
+*/ |
+typedef struct HighlightContext HighlightContext; |
+struct HighlightContext { |
+ CInstIter iter; /* Coalesced Instance Iterator */ |
+ int iPos; /* Current token offset in zIn[] */ |
+ int iRangeStart; /* First token to include */ |
+ int iRangeEnd; /* If non-zero, last token to include */ |
+ const char *zOpen; /* Opening highlight */ |
+ const char *zClose; /* Closing highlight */ |
+ const char *zIn; /* Input text */ |
+ int nIn; /* Size of input text in bytes */ |
+ int iOff; /* Current offset within zIn[] */ |
+ char *zOut; /* Output value */ |
+}; |
+ |
+/* |
+** Append text to the HighlightContext output string - p->zOut. Argument |
+** z points to a buffer containing n bytes of text to append. If n is |
+** negative, everything up until the first '\0' is appended to the output. |
+** |
+** If *pRc is set to any value other than SQLITE_OK when this function is |
+** called, it is a no-op. If an error (i.e. an OOM condition) is encountered, |
+** *pRc is set to an error code before returning. |
+*/ |
+static void fts5HighlightAppend( |
+ int *pRc, |
+ HighlightContext *p, |
+ const char *z, int n |
+){ |
+ if( *pRc==SQLITE_OK ){ |
+ if( n<0 ) n = (int)strlen(z); |
+ p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z); |
+ if( p->zOut==0 ) *pRc = SQLITE_NOMEM; |
+ } |
+} |
+ |
+/* |
+** Tokenizer callback used by implementation of highlight() function. |
+*/ |
+static int fts5HighlightCb( |
+ void *pContext, /* Pointer to HighlightContext object */ |
+ int tflags, /* Mask of FTS5_TOKEN_* flags */ |
+ const char *pToken, /* Buffer containing token */ |
+ int nToken, /* Size of token in bytes */ |
+ int iStartOff, /* Start offset of token */ |
+ int iEndOff /* End offset of token */ |
+){ |
+ HighlightContext *p = (HighlightContext*)pContext; |
+ int rc = SQLITE_OK; |
+ int iPos; |
+ |
+ UNUSED_PARAM2(pToken, nToken); |
+ |
+ if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; |
+ iPos = p->iPos++; |
+ |
+ if( p->iRangeEnd>0 ){ |
+ if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; |
+ if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; |
+ } |
+ |
+ if( iPos==p->iter.iStart ){ |
+ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff); |
+ fts5HighlightAppend(&rc, p, p->zOpen, -1); |
+ p->iOff = iStartOff; |
+ } |
+ |
+ if( iPos==p->iter.iEnd ){ |
+ if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){ |
+ fts5HighlightAppend(&rc, p, p->zOpen, -1); |
+ } |
+ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); |
+ fts5HighlightAppend(&rc, p, p->zClose, -1); |
+ p->iOff = iEndOff; |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5CInstIterNext(&p->iter); |
+ } |
+ } |
+ |
+ if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ |
+ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); |
+ p->iOff = iEndOff; |
+ if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){ |
+ fts5HighlightAppend(&rc, p, p->zClose, -1); |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Implementation of highlight() function. |
+*/ |
+static void fts5HighlightFunction( |
+ const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ |
+ Fts5Context *pFts, /* First arg to pass to pApi functions */ |
+ sqlite3_context *pCtx, /* Context for returning result/error */ |
+ int nVal, /* Number of values in apVal[] array */ |
+ sqlite3_value **apVal /* Array of trailing arguments */ |
+){ |
+ HighlightContext ctx; |
+ int rc; |
+ int iCol; |
+ |
+ if( nVal!=3 ){ |
+ const char *zErr = "wrong number of arguments to function highlight()"; |
+ sqlite3_result_error(pCtx, zErr, -1); |
+ return; |
+ } |
+ |
+ iCol = sqlite3_value_int(apVal[0]); |
+ memset(&ctx, 0, sizeof(HighlightContext)); |
+ ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); |
+ ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); |
+ rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); |
+ |
+ if( ctx.zIn ){ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); |
+ } |
+ fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); |
+ |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); |
+ } |
+ sqlite3_free(ctx.zOut); |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_result_error_code(pCtx, rc); |
+ } |
+} |
+/* |
+** End of highlight() implementation. |
+**************************************************************************/ |
+ |
+/* |
+** Context object passed to the fts5SentenceFinderCb() function. |
+*/ |
+typedef struct Fts5SFinder Fts5SFinder; |
+struct Fts5SFinder { |
+ int iPos; /* Current token position */ |
+ int nFirstAlloc; /* Allocated size of aFirst[] */ |
+ int nFirst; /* Number of entries in aFirst[] */ |
+ int *aFirst; /* Array of first token in each sentence */ |
+ const char *zDoc; /* Document being tokenized */ |
+}; |
+ |
+/* |
+** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if |
+** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an |
+** error occurs. |
+*/ |
+static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){ |
+ if( p->nFirstAlloc==p->nFirst ){ |
+ int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64; |
+ int *aNew; |
+ |
+ aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int)); |
+ if( aNew==0 ) return SQLITE_NOMEM; |
+ p->aFirst = aNew; |
+ p->nFirstAlloc = nNew; |
+ } |
+ p->aFirst[p->nFirst++] = iAdd; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** This function is an xTokenize() callback used by the auxiliary snippet() |
+** function. Its job is to identify tokens that are the first in a sentence. |
+** For each such token, an entry is added to the SFinder.aFirst[] array. |
+*/ |
+static int fts5SentenceFinderCb( |
+ void *pContext, /* Pointer to HighlightContext object */ |
+ int tflags, /* Mask of FTS5_TOKEN_* flags */ |
+ const char *pToken, /* Buffer containing token */ |
+ int nToken, /* Size of token in bytes */ |
+ int iStartOff, /* Start offset of token */ |
+ int iEndOff /* End offset of token */ |
+){ |
+ int rc = SQLITE_OK; |
+ |
+ UNUSED_PARAM2(pToken, nToken); |
+ UNUSED_PARAM(iEndOff); |
+ |
+ if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ |
+ Fts5SFinder *p = (Fts5SFinder*)pContext; |
+ if( p->iPos>0 ){ |
+ int i; |
+ char c = 0; |
+ for(i=iStartOff-1; i>=0; i--){ |
+ c = p->zDoc[i]; |
+ if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break; |
+ } |
+ if( i!=iStartOff-1 && (c=='.' || c==':') ){ |
+ rc = fts5SentenceFinderAdd(p, p->iPos); |
+ } |
+ }else{ |
+ rc = fts5SentenceFinderAdd(p, 0); |
+ } |
+ p->iPos++; |
+ } |
+ return rc; |
+} |
+ |
+static int fts5SnippetScore( |
+ const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ |
+ Fts5Context *pFts, /* First arg to pass to pApi functions */ |
+ int nDocsize, /* Size of column in tokens */ |
+ unsigned char *aSeen, /* Array with one element per query phrase */ |
+ int iCol, /* Column to score */ |
+ int iPos, /* Starting offset to score */ |
+ int nToken, /* Max tokens per snippet */ |
+ int *pnScore, /* OUT: Score */ |
+ int *piPos /* OUT: Adjusted offset */ |
+){ |
+ int rc; |
+ int i; |
+ int ip = 0; |
+ int ic = 0; |
+ int iOff = 0; |
+ int iFirst = -1; |
+ int nInst; |
+ int nScore = 0; |
+ int iLast = 0; |
+ |
+ rc = pApi->xInstCount(pFts, &nInst); |
+ for(i=0; i<nInst && rc==SQLITE_OK; i++){ |
+ rc = pApi->xInst(pFts, i, &ip, &ic, &iOff); |
+ if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){ |
+ nScore += (aSeen[ip] ? 1 : 1000); |
+ aSeen[ip] = 1; |
+ if( iFirst<0 ) iFirst = iOff; |
+ iLast = iOff + pApi->xPhraseSize(pFts, ip); |
+ } |
+ } |
+ |
+ *pnScore = nScore; |
+ if( piPos ){ |
+ int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2; |
+ if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken; |
+ if( iAdj<0 ) iAdj = 0; |
+ *piPos = iAdj; |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Implementation of snippet() function. |
+*/ |
+static void fts5SnippetFunction( |
+ const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ |
+ Fts5Context *pFts, /* First arg to pass to pApi functions */ |
+ sqlite3_context *pCtx, /* Context for returning result/error */ |
+ int nVal, /* Number of values in apVal[] array */ |
+ sqlite3_value **apVal /* Array of trailing arguments */ |
+){ |
+ HighlightContext ctx; |
+ int rc = SQLITE_OK; /* Return code */ |
+ int iCol; /* 1st argument to snippet() */ |
+ const char *zEllips; /* 4th argument to snippet() */ |
+ int nToken; /* 5th argument to snippet() */ |
+ int nInst = 0; /* Number of instance matches this row */ |
+ int i; /* Used to iterate through instances */ |
+ int nPhrase; /* Number of phrases in query */ |
+ unsigned char *aSeen; /* Array of "seen instance" flags */ |
+ int iBestCol; /* Column containing best snippet */ |
+ int iBestStart = 0; /* First token of best snippet */ |
+ int nBestScore = 0; /* Score of best snippet */ |
+ int nColSize = 0; /* Total size of iBestCol in tokens */ |
+ Fts5SFinder sFinder; /* Used to find the beginnings of sentences */ |
+ int nCol; |
+ |
+ if( nVal!=5 ){ |
+ const char *zErr = "wrong number of arguments to function snippet()"; |
+ sqlite3_result_error(pCtx, zErr, -1); |
+ return; |
+ } |
+ |
+ nCol = pApi->xColumnCount(pFts); |
+ memset(&ctx, 0, sizeof(HighlightContext)); |
+ iCol = sqlite3_value_int(apVal[0]); |
+ ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); |
+ ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); |
+ zEllips = (const char*)sqlite3_value_text(apVal[3]); |
+ nToken = sqlite3_value_int(apVal[4]); |
+ |
+ iBestCol = (iCol>=0 ? iCol : 0); |
+ nPhrase = pApi->xPhraseCount(pFts); |
+ aSeen = sqlite3_malloc(nPhrase); |
+ if( aSeen==0 ){ |
+ rc = SQLITE_NOMEM; |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = pApi->xInstCount(pFts, &nInst); |
+ } |
+ |
+ memset(&sFinder, 0, sizeof(Fts5SFinder)); |
+ for(i=0; i<nCol; i++){ |
+ if( iCol<0 || iCol==i ){ |
+ int nDoc; |
+ int nDocsize; |
+ int ii; |
+ sFinder.iPos = 0; |
+ sFinder.nFirst = 0; |
+ rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc); |
+ if( rc!=SQLITE_OK ) break; |
+ rc = pApi->xTokenize(pFts, |
+ sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb |
+ ); |
+ if( rc!=SQLITE_OK ) break; |
+ rc = pApi->xColumnSize(pFts, i, &nDocsize); |
+ if( rc!=SQLITE_OK ) break; |
+ |
+ for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){ |
+ int ip, ic, io; |
+ int iAdj; |
+ int nScore; |
+ int jj; |
+ |
+ rc = pApi->xInst(pFts, ii, &ip, &ic, &io); |
+ if( ic!=i || rc!=SQLITE_OK ) continue; |
+ memset(aSeen, 0, nPhrase); |
+ rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, |
+ io, nToken, &nScore, &iAdj |
+ ); |
+ if( rc==SQLITE_OK && nScore>nBestScore ){ |
+ nBestScore = nScore; |
+ iBestCol = i; |
+ iBestStart = iAdj; |
+ nColSize = nDocsize; |
+ } |
+ |
+ if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){ |
+ for(jj=0; jj<(sFinder.nFirst-1); jj++){ |
+ if( sFinder.aFirst[jj+1]>io ) break; |
+ } |
+ |
+ if( sFinder.aFirst[jj]<io ){ |
+ memset(aSeen, 0, nPhrase); |
+ rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, |
+ sFinder.aFirst[jj], nToken, &nScore, 0 |
+ ); |
+ |
+ nScore += (sFinder.aFirst[jj]==0 ? 120 : 100); |
+ if( rc==SQLITE_OK && nScore>nBestScore ){ |
+ nBestScore = nScore; |
+ iBestCol = i; |
+ iBestStart = sFinder.aFirst[jj]; |
+ nColSize = nDocsize; |
+ } |
+ } |
+ } |
+ } |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); |
+ } |
+ if( rc==SQLITE_OK && nColSize==0 ){ |
+ rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); |
+ } |
+ if( ctx.zIn ){ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); |
+ } |
+ |
+ ctx.iRangeStart = iBestStart; |
+ ctx.iRangeEnd = iBestStart + nToken - 1; |
+ |
+ if( iBestStart>0 ){ |
+ fts5HighlightAppend(&rc, &ctx, zEllips, -1); |
+ } |
+ |
+ /* Advance iterator ctx.iter so that it points to the first coalesced |
+ ** phrase instance at or following position iBestStart. */ |
+ while( ctx.iter.iStart>=0 && ctx.iter.iStart<iBestStart && rc==SQLITE_OK ){ |
+ rc = fts5CInstIterNext(&ctx.iter); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); |
+ } |
+ if( ctx.iRangeEnd>=(nColSize-1) ){ |
+ fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); |
+ }else{ |
+ fts5HighlightAppend(&rc, &ctx, zEllips, -1); |
+ } |
+ } |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); |
+ }else{ |
+ sqlite3_result_error_code(pCtx, rc); |
+ } |
+ sqlite3_free(ctx.zOut); |
+ sqlite3_free(aSeen); |
+ sqlite3_free(sFinder.aFirst); |
+} |
+ |
+/************************************************************************/ |
+ |
+/* |
+** The first time the bm25() function is called for a query, an instance |
+** of the following structure is allocated and populated. |
+*/ |
+typedef struct Fts5Bm25Data Fts5Bm25Data; |
+struct Fts5Bm25Data { |
+ int nPhrase; /* Number of phrases in query */ |
+ double avgdl; /* Average number of tokens in each row */ |
+ double *aIDF; /* IDF for each phrase */ |
+ double *aFreq; /* Array used to calculate phrase freq. */ |
+}; |
+ |
+/* |
+** Callback used by fts5Bm25GetData() to count the number of rows in the |
+** table matched by each individual phrase within the query. |
+*/ |
+static int fts5CountCb( |
+ const Fts5ExtensionApi *pApi, |
+ Fts5Context *pFts, |
+ void *pUserData /* Pointer to sqlite3_int64 variable */ |
+){ |
+ sqlite3_int64 *pn = (sqlite3_int64*)pUserData; |
+ UNUSED_PARAM2(pApi, pFts); |
+ (*pn)++; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Set *ppData to point to the Fts5Bm25Data object for the current query. |
+** If the object has not already been allocated, allocate and populate it |
+** now. |
+*/ |
+static int fts5Bm25GetData( |
+ const Fts5ExtensionApi *pApi, |
+ Fts5Context *pFts, |
+ Fts5Bm25Data **ppData /* OUT: bm25-data object for this query */ |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ Fts5Bm25Data *p; /* Object to return */ |
+ |
+ p = pApi->xGetAuxdata(pFts, 0); |
+ if( p==0 ){ |
+ int nPhrase; /* Number of phrases in query */ |
+ sqlite3_int64 nRow = 0; /* Number of rows in table */ |
+ sqlite3_int64 nToken = 0; /* Number of tokens in table */ |
+ int nByte; /* Bytes of space to allocate */ |
+ int i; |
+ |
+ /* Allocate the Fts5Bm25Data object */ |
+ nPhrase = pApi->xPhraseCount(pFts); |
+ nByte = sizeof(Fts5Bm25Data) + nPhrase*2*sizeof(double); |
+ p = (Fts5Bm25Data*)sqlite3_malloc(nByte); |
+ if( p==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ memset(p, 0, nByte); |
+ p->nPhrase = nPhrase; |
+ p->aIDF = (double*)&p[1]; |
+ p->aFreq = &p->aIDF[nPhrase]; |
+ } |
+ |
+ /* Calculate the average document length for this FTS5 table */ |
+ if( rc==SQLITE_OK ) rc = pApi->xRowCount(pFts, &nRow); |
+ if( rc==SQLITE_OK ) rc = pApi->xColumnTotalSize(pFts, -1, &nToken); |
+ if( rc==SQLITE_OK ) p->avgdl = (double)nToken / (double)nRow; |
+ |
+ /* Calculate an IDF for each phrase in the query */ |
+ for(i=0; rc==SQLITE_OK && i<nPhrase; i++){ |
+ sqlite3_int64 nHit = 0; |
+ rc = pApi->xQueryPhrase(pFts, i, (void*)&nHit, fts5CountCb); |
+ if( rc==SQLITE_OK ){ |
+ /* Calculate the IDF (Inverse Document Frequency) for phrase i. |
+ ** This is done using the standard BM25 formula as found on wikipedia: |
+ ** |
+ ** IDF = log( (N - nHit + 0.5) / (nHit + 0.5) ) |
+ ** |
+ ** where "N" is the total number of documents in the set and nHit |
+ ** is the number that contain at least one instance of the phrase |
+ ** under consideration. |
+ ** |
+ ** The problem with this is that if (N < 2*nHit), the IDF is |
+ ** negative. Which is undesirable. So the mimimum allowable IDF is |
+ ** (1e-6) - roughly the same as a term that appears in just over |
+ ** half of set of 5,000,000 documents. */ |
+ double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) ); |
+ if( idf<=0.0 ) idf = 1e-6; |
+ p->aIDF[i] = idf; |
+ } |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_free(p); |
+ }else{ |
+ rc = pApi->xSetAuxdata(pFts, p, sqlite3_free); |
+ } |
+ if( rc!=SQLITE_OK ) p = 0; |
+ } |
+ *ppData = p; |
+ return rc; |
+} |
+ |
+/* |
+** Implementation of bm25() function. |
+*/ |
+static void fts5Bm25Function( |
+ const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ |
+ Fts5Context *pFts, /* First arg to pass to pApi functions */ |
+ sqlite3_context *pCtx, /* Context for returning result/error */ |
+ int nVal, /* Number of values in apVal[] array */ |
+ sqlite3_value **apVal /* Array of trailing arguments */ |
+){ |
+ const double k1 = 1.2; /* Constant "k1" from BM25 formula */ |
+ const double b = 0.75; /* Constant "b" from BM25 formula */ |
+ int rc = SQLITE_OK; /* Error code */ |
+ double score = 0.0; /* SQL function return value */ |
+ Fts5Bm25Data *pData; /* Values allocated/calculated once only */ |
+ int i; /* Iterator variable */ |
+ int nInst = 0; /* Value returned by xInstCount() */ |
+ double D = 0.0; /* Total number of tokens in row */ |
+ double *aFreq = 0; /* Array of phrase freq. for current row */ |
+ |
+ /* Calculate the phrase frequency (symbol "f(qi,D)" in the documentation) |
+ ** for each phrase in the query for the current row. */ |
+ rc = fts5Bm25GetData(pApi, pFts, &pData); |
+ if( rc==SQLITE_OK ){ |
+ aFreq = pData->aFreq; |
+ memset(aFreq, 0, sizeof(double) * pData->nPhrase); |
+ rc = pApi->xInstCount(pFts, &nInst); |
+ } |
+ for(i=0; rc==SQLITE_OK && i<nInst; i++){ |
+ int ip; int ic; int io; |
+ rc = pApi->xInst(pFts, i, &ip, &ic, &io); |
+ if( rc==SQLITE_OK ){ |
+ double w = (nVal > ic) ? sqlite3_value_double(apVal[ic]) : 1.0; |
+ aFreq[ip] += w; |
+ } |
+ } |
+ |
+ /* Figure out the total size of the current row in tokens. */ |
+ if( rc==SQLITE_OK ){ |
+ int nTok; |
+ rc = pApi->xColumnSize(pFts, -1, &nTok); |
+ D = (double)nTok; |
+ } |
+ |
+ /* Determine the BM25 score for the current row. */ |
+ for(i=0; rc==SQLITE_OK && i<pData->nPhrase; i++){ |
+ score += pData->aIDF[i] * ( |
+ ( aFreq[i] * (k1 + 1.0) ) / |
+ ( aFreq[i] + k1 * (1 - b + b * D / pData->avgdl) ) |
+ ); |
+ } |
+ |
+ /* If no error has occurred, return the calculated score. Otherwise, |
+ ** throw an SQL exception. */ |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_result_double(pCtx, -1.0 * score); |
+ }else{ |
+ sqlite3_result_error_code(pCtx, rc); |
+ } |
+} |
+ |
+static int sqlite3Fts5AuxInit(fts5_api *pApi){ |
+ struct Builtin { |
+ const char *zFunc; /* Function name (nul-terminated) */ |
+ void *pUserData; /* User-data pointer */ |
+ fts5_extension_function xFunc;/* Callback function */ |
+ void (*xDestroy)(void*); /* Destructor function */ |
+ } aBuiltin [] = { |
+ { "snippet", 0, fts5SnippetFunction, 0 }, |
+ { "highlight", 0, fts5HighlightFunction, 0 }, |
+ { "bm25", 0, fts5Bm25Function, 0 }, |
+ }; |
+ int rc = SQLITE_OK; /* Return code */ |
+ int i; /* To iterate through builtin functions */ |
+ |
+ for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){ |
+ rc = pApi->xCreateFunction(pApi, |
+ aBuiltin[i].zFunc, |
+ aBuiltin[i].pUserData, |
+ aBuiltin[i].xFunc, |
+ aBuiltin[i].xDestroy |
+ ); |
+ } |
+ |
+ return rc; |
+} |
+ |
+ |
+ |
+/* |
+** 2014 May 31 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+*/ |
+ |
+ |
+ |
+/* #include "fts5Int.h" */ |
+ |
+static int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){ |
+ if( (u32)pBuf->nSpace<nByte ){ |
+ u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64; |
+ u8 *pNew; |
+ while( nNew<nByte ){ |
+ nNew = nNew * 2; |
+ } |
+ pNew = sqlite3_realloc(pBuf->p, nNew); |
+ if( pNew==0 ){ |
+ *pRc = SQLITE_NOMEM; |
+ return 1; |
+ }else{ |
+ pBuf->nSpace = nNew; |
+ pBuf->p = pNew; |
+ } |
+ } |
+ return 0; |
+} |
+ |
+ |
+/* |
+** Encode value iVal as an SQLite varint and append it to the buffer object |
+** pBuf. If an OOM error occurs, set the error code in p. |
+*/ |
+static void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){ |
+ if( fts5BufferGrow(pRc, pBuf, 9) ) return; |
+ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal); |
+} |
+ |
+static void sqlite3Fts5Put32(u8 *aBuf, int iVal){ |
+ aBuf[0] = (iVal>>24) & 0x00FF; |
+ aBuf[1] = (iVal>>16) & 0x00FF; |
+ aBuf[2] = (iVal>> 8) & 0x00FF; |
+ aBuf[3] = (iVal>> 0) & 0x00FF; |
+} |
+ |
+static int sqlite3Fts5Get32(const u8 *aBuf){ |
+ return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3]; |
+} |
+ |
+/* |
+** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set |
+** the error code in p. If an error has already occurred when this function |
+** is called, it is a no-op. |
+*/ |
+static void sqlite3Fts5BufferAppendBlob( |
+ int *pRc, |
+ Fts5Buffer *pBuf, |
+ u32 nData, |
+ const u8 *pData |
+){ |
+ assert_nc( *pRc || nData>=0 ); |
+ if( fts5BufferGrow(pRc, pBuf, nData) ) return; |
+ memcpy(&pBuf->p[pBuf->n], pData, nData); |
+ pBuf->n += nData; |
+} |
+ |
+/* |
+** Append the nul-terminated string zStr to the buffer pBuf. This function |
+** ensures that the byte following the buffer data is set to 0x00, even |
+** though this byte is not included in the pBuf->n count. |
+*/ |
+static void sqlite3Fts5BufferAppendString( |
+ int *pRc, |
+ Fts5Buffer *pBuf, |
+ const char *zStr |
+){ |
+ int nStr = (int)strlen(zStr); |
+ sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr+1, (const u8*)zStr); |
+ pBuf->n--; |
+} |
+ |
+/* |
+** Argument zFmt is a printf() style format string. This function performs |
+** the printf() style processing, then appends the results to buffer pBuf. |
+** |
+** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte |
+** following the buffer data is set to 0x00, even though this byte is not |
+** included in the pBuf->n count. |
+*/ |
+static void sqlite3Fts5BufferAppendPrintf( |
+ int *pRc, |
+ Fts5Buffer *pBuf, |
+ char *zFmt, ... |
+){ |
+ if( *pRc==SQLITE_OK ){ |
+ char *zTmp; |
+ va_list ap; |
+ va_start(ap, zFmt); |
+ zTmp = sqlite3_vmprintf(zFmt, ap); |
+ va_end(ap); |
+ |
+ if( zTmp==0 ){ |
+ *pRc = SQLITE_NOMEM; |
+ }else{ |
+ sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp); |
+ sqlite3_free(zTmp); |
+ } |
+ } |
+} |
+ |
+static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...){ |
+ char *zRet = 0; |
+ if( *pRc==SQLITE_OK ){ |
+ va_list ap; |
+ va_start(ap, zFmt); |
+ zRet = sqlite3_vmprintf(zFmt, ap); |
+ va_end(ap); |
+ if( zRet==0 ){ |
+ *pRc = SQLITE_NOMEM; |
+ } |
+ } |
+ return zRet; |
+} |
+ |
+ |
+/* |
+** Free any buffer allocated by pBuf. Zero the structure before returning. |
+*/ |
+static void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){ |
+ sqlite3_free(pBuf->p); |
+ memset(pBuf, 0, sizeof(Fts5Buffer)); |
+} |
+ |
+/* |
+** Zero the contents of the buffer object. But do not free the associated |
+** memory allocation. |
+*/ |
+static void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){ |
+ pBuf->n = 0; |
+} |
+ |
+/* |
+** Set the buffer to contain nData/pData. If an OOM error occurs, leave an |
+** the error code in p. If an error has already occurred when this function |
+** is called, it is a no-op. |
+*/ |
+static void sqlite3Fts5BufferSet( |
+ int *pRc, |
+ Fts5Buffer *pBuf, |
+ int nData, |
+ const u8 *pData |
+){ |
+ pBuf->n = 0; |
+ sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData); |
+} |
+ |
+static int sqlite3Fts5PoslistNext64( |
+ const u8 *a, int n, /* Buffer containing poslist */ |
+ int *pi, /* IN/OUT: Offset within a[] */ |
+ i64 *piOff /* IN/OUT: Current offset */ |
+){ |
+ int i = *pi; |
+ if( i>=n ){ |
+ /* EOF */ |
+ *piOff = -1; |
+ return 1; |
+ }else{ |
+ i64 iOff = *piOff; |
+ int iVal; |
+ fts5FastGetVarint32(a, i, iVal); |
+ if( iVal==1 ){ |
+ fts5FastGetVarint32(a, i, iVal); |
+ iOff = ((i64)iVal) << 32; |
+ fts5FastGetVarint32(a, i, iVal); |
+ } |
+ *piOff = iOff + (iVal-2); |
+ *pi = i; |
+ return 0; |
+ } |
+} |
+ |
+ |
+/* |
+** Advance the iterator object passed as the only argument. Return true |
+** if the iterator reaches EOF, or false otherwise. |
+*/ |
+static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){ |
+ if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) ){ |
+ pIter->bEof = 1; |
+ } |
+ return pIter->bEof; |
+} |
+ |
+static int sqlite3Fts5PoslistReaderInit( |
+ const u8 *a, int n, /* Poslist buffer to iterate through */ |
+ Fts5PoslistReader *pIter /* Iterator object to initialize */ |
+){ |
+ memset(pIter, 0, sizeof(*pIter)); |
+ pIter->a = a; |
+ pIter->n = n; |
+ sqlite3Fts5PoslistReaderNext(pIter); |
+ return pIter->bEof; |
+} |
+ |
+/* |
+** Append position iPos to the position list being accumulated in buffer |
+** pBuf, which must be already be large enough to hold the new data. |
+** The previous position written to this list is *piPrev. *piPrev is set |
+** to iPos before returning. |
+*/ |
+static void sqlite3Fts5PoslistSafeAppend( |
+ Fts5Buffer *pBuf, |
+ i64 *piPrev, |
+ i64 iPos |
+){ |
+ static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; |
+ if( (iPos & colmask) != (*piPrev & colmask) ){ |
+ pBuf->p[pBuf->n++] = 1; |
+ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); |
+ *piPrev = (iPos & colmask); |
+ } |
+ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2); |
+ *piPrev = iPos; |
+} |
+ |
+static int sqlite3Fts5PoslistWriterAppend( |
+ Fts5Buffer *pBuf, |
+ Fts5PoslistWriter *pWriter, |
+ i64 iPos |
+){ |
+ int rc = 0; /* Initialized only to suppress erroneous warning from Clang */ |
+ if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc; |
+ sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos); |
+ return SQLITE_OK; |
+} |
+ |
+static void *sqlite3Fts5MallocZero(int *pRc, int nByte){ |
+ void *pRet = 0; |
+ if( *pRc==SQLITE_OK ){ |
+ pRet = sqlite3_malloc(nByte); |
+ if( pRet==0 && nByte>0 ){ |
+ *pRc = SQLITE_NOMEM; |
+ }else{ |
+ memset(pRet, 0, nByte); |
+ } |
+ } |
+ return pRet; |
+} |
+ |
+/* |
+** Return a nul-terminated copy of the string indicated by pIn. If nIn |
+** is non-negative, then it is the length of the string in bytes. Otherwise, |
+** the length of the string is determined using strlen(). |
+** |
+** It is the responsibility of the caller to eventually free the returned |
+** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. |
+*/ |
+static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ |
+ char *zRet = 0; |
+ if( *pRc==SQLITE_OK ){ |
+ if( nIn<0 ){ |
+ nIn = (int)strlen(pIn); |
+ } |
+ zRet = (char*)sqlite3_malloc(nIn+1); |
+ if( zRet ){ |
+ memcpy(zRet, pIn, nIn); |
+ zRet[nIn] = '\0'; |
+ }else{ |
+ *pRc = SQLITE_NOMEM; |
+ } |
+ } |
+ return zRet; |
+} |
+ |
+ |
+/* |
+** Return true if character 't' may be part of an FTS5 bareword, or false |
+** otherwise. Characters that may be part of barewords: |
+** |
+** * All non-ASCII characters, |
+** * The 52 upper and lower case ASCII characters, and |
+** * The 10 integer ASCII characters. |
+** * The underscore character "_" (0x5F). |
+** * The unicode "subsitute" character (0x1A). |
+*/ |
+static int sqlite3Fts5IsBareword(char t){ |
+ u8 aBareword[128] = { |
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */ |
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */ |
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */ |
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */ |
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */ |
+ }; |
+ |
+ return (t & 0x80) || aBareword[(int)t]; |
+} |
+ |
+ |
+/************************************************************************* |
+*/ |
+typedef struct Fts5TermsetEntry Fts5TermsetEntry; |
+struct Fts5TermsetEntry { |
+ char *pTerm; |
+ int nTerm; |
+ int iIdx; /* Index (main or aPrefix[] entry) */ |
+ Fts5TermsetEntry *pNext; |
+}; |
+ |
+struct Fts5Termset { |
+ Fts5TermsetEntry *apHash[512]; |
+}; |
+ |
+static int sqlite3Fts5TermsetNew(Fts5Termset **pp){ |
+ int rc = SQLITE_OK; |
+ *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset)); |
+ return rc; |
+} |
+ |
+static int sqlite3Fts5TermsetAdd( |
+ Fts5Termset *p, |
+ int iIdx, |
+ const char *pTerm, int nTerm, |
+ int *pbPresent |
+){ |
+ int rc = SQLITE_OK; |
+ *pbPresent = 0; |
+ if( p ){ |
+ int i; |
+ u32 hash = 13; |
+ Fts5TermsetEntry *pEntry; |
+ |
+ /* Calculate a hash value for this term. This is the same hash checksum |
+ ** used by the fts5_hash.c module. This is not important for correct |
+ ** operation of the module, but is necessary to ensure that some tests |
+ ** designed to produce hash table collisions really do work. */ |
+ for(i=nTerm-1; i>=0; i--){ |
+ hash = (hash << 3) ^ hash ^ pTerm[i]; |
+ } |
+ hash = (hash << 3) ^ hash ^ iIdx; |
+ hash = hash % ArraySize(p->apHash); |
+ |
+ for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ |
+ if( pEntry->iIdx==iIdx |
+ && pEntry->nTerm==nTerm |
+ && memcmp(pEntry->pTerm, pTerm, nTerm)==0 |
+ ){ |
+ *pbPresent = 1; |
+ break; |
+ } |
+ } |
+ |
+ if( pEntry==0 ){ |
+ pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm); |
+ if( pEntry ){ |
+ pEntry->pTerm = (char*)&pEntry[1]; |
+ pEntry->nTerm = nTerm; |
+ pEntry->iIdx = iIdx; |
+ memcpy(pEntry->pTerm, pTerm, nTerm); |
+ pEntry->pNext = p->apHash[hash]; |
+ p->apHash[hash] = pEntry; |
+ } |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+static void sqlite3Fts5TermsetFree(Fts5Termset *p){ |
+ if( p ){ |
+ u32 i; |
+ for(i=0; i<ArraySize(p->apHash); i++){ |
+ Fts5TermsetEntry *pEntry = p->apHash[i]; |
+ while( pEntry ){ |
+ Fts5TermsetEntry *pDel = pEntry; |
+ pEntry = pEntry->pNext; |
+ sqlite3_free(pDel); |
+ } |
+ } |
+ sqlite3_free(p); |
+ } |
+} |
+ |
+/* |
+** 2014 Jun 09 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** This is an SQLite module implementing full-text search. |
+*/ |
+ |
+ |
+/* #include "fts5Int.h" */ |
+ |
+#define FTS5_DEFAULT_PAGE_SIZE 4050 |
+#define FTS5_DEFAULT_AUTOMERGE 4 |
+#define FTS5_DEFAULT_USERMERGE 4 |
+#define FTS5_DEFAULT_CRISISMERGE 16 |
+#define FTS5_DEFAULT_HASHSIZE (1024*1024) |
+ |
+/* Maximum allowed page size */ |
+#define FTS5_MAX_PAGE_SIZE (128*1024) |
+ |
+static int fts5_iswhitespace(char x){ |
+ return (x==' '); |
+} |
+ |
+static int fts5_isopenquote(char x){ |
+ return (x=='"' || x=='\'' || x=='[' || x=='`'); |
+} |
+ |
+/* |
+** Argument pIn points to a character that is part of a nul-terminated |
+** string. Return a pointer to the first character following *pIn in |
+** the string that is not a white-space character. |
+*/ |
+static const char *fts5ConfigSkipWhitespace(const char *pIn){ |
+ const char *p = pIn; |
+ if( p ){ |
+ while( fts5_iswhitespace(*p) ){ p++; } |
+ } |
+ return p; |
+} |
+ |
+/* |
+** Argument pIn points to a character that is part of a nul-terminated |
+** string. Return a pointer to the first character following *pIn in |
+** the string that is not a "bareword" character. |
+*/ |
+static const char *fts5ConfigSkipBareword(const char *pIn){ |
+ const char *p = pIn; |
+ while ( sqlite3Fts5IsBareword(*p) ) p++; |
+ if( p==pIn ) p = 0; |
+ return p; |
+} |
+ |
+static int fts5_isdigit(char a){ |
+ return (a>='0' && a<='9'); |
+} |
+ |
+ |
+ |
+static const char *fts5ConfigSkipLiteral(const char *pIn){ |
+ const char *p = pIn; |
+ switch( *p ){ |
+ case 'n': case 'N': |
+ if( sqlite3_strnicmp("null", p, 4)==0 ){ |
+ p = &p[4]; |
+ }else{ |
+ p = 0; |
+ } |
+ break; |
+ |
+ case 'x': case 'X': |
+ p++; |
+ if( *p=='\'' ){ |
+ p++; |
+ while( (*p>='a' && *p<='f') |
+ || (*p>='A' && *p<='F') |
+ || (*p>='0' && *p<='9') |
+ ){ |
+ p++; |
+ } |
+ if( *p=='\'' && 0==((p-pIn)%2) ){ |
+ p++; |
+ }else{ |
+ p = 0; |
+ } |
+ }else{ |
+ p = 0; |
+ } |
+ break; |
+ |
+ case '\'': |
+ p++; |
+ while( p ){ |
+ if( *p=='\'' ){ |
+ p++; |
+ if( *p!='\'' ) break; |
+ } |
+ p++; |
+ if( *p==0 ) p = 0; |
+ } |
+ break; |
+ |
+ default: |
+ /* maybe a number */ |
+ if( *p=='+' || *p=='-' ) p++; |
+ while( fts5_isdigit(*p) ) p++; |
+ |
+ /* At this point, if the literal was an integer, the parse is |
+ ** finished. Or, if it is a floating point value, it may continue |
+ ** with either a decimal point or an 'E' character. */ |
+ if( *p=='.' && fts5_isdigit(p[1]) ){ |
+ p += 2; |
+ while( fts5_isdigit(*p) ) p++; |
+ } |
+ if( p==pIn ) p = 0; |
+ |
+ break; |
+ } |
+ |
+ return p; |
+} |
+ |
+/* |
+** The first character of the string pointed to by argument z is guaranteed |
+** to be an open-quote character (see function fts5_isopenquote()). |
+** |
+** This function searches for the corresponding close-quote character within |
+** the string and, if found, dequotes the string in place and adds a new |
+** nul-terminator byte. |
+** |
+** If the close-quote is found, the value returned is the byte offset of |
+** the character immediately following it. Or, if the close-quote is not |
+** found, -1 is returned. If -1 is returned, the buffer is left in an |
+** undefined state. |
+*/ |
+static int fts5Dequote(char *z){ |
+ char q; |
+ int iIn = 1; |
+ int iOut = 0; |
+ q = z[0]; |
+ |
+ /* Set stack variable q to the close-quote character */ |
+ assert( q=='[' || q=='\'' || q=='"' || q=='`' ); |
+ if( q=='[' ) q = ']'; |
+ |
+ while( ALWAYS(z[iIn]) ){ |
+ if( z[iIn]==q ){ |
+ if( z[iIn+1]!=q ){ |
+ /* Character iIn was the close quote. */ |
+ iIn++; |
+ break; |
+ }else{ |
+ /* Character iIn and iIn+1 form an escaped quote character. Skip |
+ ** the input cursor past both and copy a single quote character |
+ ** to the output buffer. */ |
+ iIn += 2; |
+ z[iOut++] = q; |
+ } |
+ }else{ |
+ z[iOut++] = z[iIn++]; |
+ } |
+ } |
+ |
+ z[iOut] = '\0'; |
+ return iIn; |
+} |
+ |
+/* |
+** Convert an SQL-style quoted string into a normal string by removing |
+** the quote characters. The conversion is done in-place. If the |
+** input does not begin with a quote character, then this routine |
+** is a no-op. |
+** |
+** Examples: |
+** |
+** "abc" becomes abc |
+** 'xyz' becomes xyz |
+** [pqr] becomes pqr |
+** `mno` becomes mno |
+*/ |
+static void sqlite3Fts5Dequote(char *z){ |
+ char quote; /* Quote character (if any ) */ |
+ |
+ assert( 0==fts5_iswhitespace(z[0]) ); |
+ quote = z[0]; |
+ if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ |
+ fts5Dequote(z); |
+ } |
+} |
+ |
+ |
+struct Fts5Enum { |
+ const char *zName; |
+ int eVal; |
+}; |
+typedef struct Fts5Enum Fts5Enum; |
+ |
+static int fts5ConfigSetEnum( |
+ const Fts5Enum *aEnum, |
+ const char *zEnum, |
+ int *peVal |
+){ |
+ int nEnum = (int)strlen(zEnum); |
+ int i; |
+ int iVal = -1; |
+ |
+ for(i=0; aEnum[i].zName; i++){ |
+ if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){ |
+ if( iVal>=0 ) return SQLITE_ERROR; |
+ iVal = aEnum[i].eVal; |
+ } |
+ } |
+ |
+ *peVal = iVal; |
+ return iVal<0 ? SQLITE_ERROR : SQLITE_OK; |
+} |
+ |
+/* |
+** Parse a "special" CREATE VIRTUAL TABLE directive and update |
+** configuration object pConfig as appropriate. |
+** |
+** If successful, object pConfig is updated and SQLITE_OK returned. If |
+** an error occurs, an SQLite error code is returned and an error message |
+** may be left in *pzErr. It is the responsibility of the caller to |
+** eventually free any such error message using sqlite3_free(). |
+*/ |
+static int fts5ConfigParseSpecial( |
+ Fts5Global *pGlobal, |
+ Fts5Config *pConfig, /* Configuration object to update */ |
+ const char *zCmd, /* Special command to parse */ |
+ const char *zArg, /* Argument to parse */ |
+ char **pzErr /* OUT: Error message */ |
+){ |
+ int rc = SQLITE_OK; |
+ int nCmd = (int)strlen(zCmd); |
+ if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){ |
+ const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES; |
+ const char *p; |
+ int bFirst = 1; |
+ if( pConfig->aPrefix==0 ){ |
+ pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte); |
+ if( rc ) return rc; |
+ } |
+ |
+ p = zArg; |
+ while( 1 ){ |
+ int nPre = 0; |
+ |
+ while( p[0]==' ' ) p++; |
+ if( bFirst==0 && p[0]==',' ){ |
+ p++; |
+ while( p[0]==' ' ) p++; |
+ }else if( p[0]=='\0' ){ |
+ break; |
+ } |
+ if( p[0]<'0' || p[0]>'9' ){ |
+ *pzErr = sqlite3_mprintf("malformed prefix=... directive"); |
+ rc = SQLITE_ERROR; |
+ break; |
+ } |
+ |
+ if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES ){ |
+ *pzErr = sqlite3_mprintf( |
+ "too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES |
+ ); |
+ rc = SQLITE_ERROR; |
+ break; |
+ } |
+ |
+ while( p[0]>='0' && p[0]<='9' && nPre<1000 ){ |
+ nPre = nPre*10 + (p[0] - '0'); |
+ p++; |
+ } |
+ |
+ if( nPre<=0 || nPre>=1000 ){ |
+ *pzErr = sqlite3_mprintf("prefix length out of range (max 999)"); |
+ rc = SQLITE_ERROR; |
+ break; |
+ } |
+ |
+ pConfig->aPrefix[pConfig->nPrefix] = nPre; |
+ pConfig->nPrefix++; |
+ bFirst = 0; |
+ } |
+ assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES ); |
+ return rc; |
+ } |
+ |
+ if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){ |
+ const char *p = (const char*)zArg; |
+ int nArg = (int)strlen(zArg) + 1; |
+ char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg); |
+ char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2); |
+ char *pSpace = pDel; |
+ |
+ if( azArg && pSpace ){ |
+ if( pConfig->pTok ){ |
+ *pzErr = sqlite3_mprintf("multiple tokenize=... directives"); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ for(nArg=0; p && *p; nArg++){ |
+ const char *p2 = fts5ConfigSkipWhitespace(p); |
+ if( *p2=='\'' ){ |
+ p = fts5ConfigSkipLiteral(p2); |
+ }else{ |
+ p = fts5ConfigSkipBareword(p2); |
+ } |
+ if( p ){ |
+ memcpy(pSpace, p2, p-p2); |
+ azArg[nArg] = pSpace; |
+ sqlite3Fts5Dequote(pSpace); |
+ pSpace += (p - p2) + 1; |
+ p = fts5ConfigSkipWhitespace(p); |
+ } |
+ } |
+ if( p==0 ){ |
+ *pzErr = sqlite3_mprintf("parse error in tokenize directive"); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ rc = sqlite3Fts5GetTokenizer(pGlobal, |
+ (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi, |
+ pzErr |
+ ); |
+ } |
+ } |
+ } |
+ |
+ sqlite3_free(azArg); |
+ sqlite3_free(pDel); |
+ return rc; |
+ } |
+ |
+ if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){ |
+ if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ |
+ *pzErr = sqlite3_mprintf("multiple content=... directives"); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ if( zArg[0] ){ |
+ pConfig->eContent = FTS5_CONTENT_EXTERNAL; |
+ pConfig->zContent = sqlite3Fts5Mprintf(&rc, "%Q.%Q", pConfig->zDb,zArg); |
+ }else{ |
+ pConfig->eContent = FTS5_CONTENT_NONE; |
+ } |
+ } |
+ return rc; |
+ } |
+ |
+ if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ |
+ if( pConfig->zContentRowid ){ |
+ *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1); |
+ } |
+ return rc; |
+ } |
+ |
+ if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){ |
+ if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ |
+ *pzErr = sqlite3_mprintf("malformed columnsize=... directive"); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ pConfig->bColumnsize = (zArg[0]=='1'); |
+ } |
+ return rc; |
+ } |
+ |
+ if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ |
+ const Fts5Enum aDetail[] = { |
+ { "none", FTS5_DETAIL_NONE }, |
+ { "full", FTS5_DETAIL_FULL }, |
+ { "columns", FTS5_DETAIL_COLUMNS }, |
+ { 0, 0 } |
+ }; |
+ |
+ if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){ |
+ *pzErr = sqlite3_mprintf("malformed detail=... directive"); |
+ } |
+ return rc; |
+ } |
+ |
+ *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); |
+ return SQLITE_ERROR; |
+} |
+ |
+/* |
+** Allocate an instance of the default tokenizer ("simple") at |
+** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error |
+** code if an error occurs. |
+*/ |
+static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){ |
+ assert( pConfig->pTok==0 && pConfig->pTokApi==0 ); |
+ return sqlite3Fts5GetTokenizer( |
+ pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0 |
+ ); |
+} |
+ |
+/* |
+** Gobble up the first bareword or quoted word from the input buffer zIn. |
+** Return a pointer to the character immediately following the last in |
+** the gobbled word if successful, or a NULL pointer otherwise (failed |
+** to find close-quote character). |
+** |
+** Before returning, set pzOut to point to a new buffer containing a |
+** nul-terminated, dequoted copy of the gobbled word. If the word was |
+** quoted, *pbQuoted is also set to 1 before returning. |
+** |
+** If *pRc is other than SQLITE_OK when this function is called, it is |
+** a no-op (NULL is returned). Otherwise, if an OOM occurs within this |
+** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not* |
+** set if a parse error (failed to find close quote) occurs. |
+*/ |
+static const char *fts5ConfigGobbleWord( |
+ int *pRc, /* IN/OUT: Error code */ |
+ const char *zIn, /* Buffer to gobble string/bareword from */ |
+ char **pzOut, /* OUT: malloc'd buffer containing str/bw */ |
+ int *pbQuoted /* OUT: Set to true if dequoting required */ |
+){ |
+ const char *zRet = 0; |
+ |
+ int nIn = (int)strlen(zIn); |
+ char *zOut = sqlite3_malloc(nIn+1); |
+ |
+ assert( *pRc==SQLITE_OK ); |
+ *pbQuoted = 0; |
+ *pzOut = 0; |
+ |
+ if( zOut==0 ){ |
+ *pRc = SQLITE_NOMEM; |
+ }else{ |
+ memcpy(zOut, zIn, nIn+1); |
+ if( fts5_isopenquote(zOut[0]) ){ |
+ int ii = fts5Dequote(zOut); |
+ zRet = &zIn[ii]; |
+ *pbQuoted = 1; |
+ }else{ |
+ zRet = fts5ConfigSkipBareword(zIn); |
+ if( zRet ){ |
+ zOut[zRet-zIn] = '\0'; |
+ } |
+ } |
+ } |
+ |
+ if( zRet==0 ){ |
+ sqlite3_free(zOut); |
+ }else{ |
+ *pzOut = zOut; |
+ } |
+ |
+ return zRet; |
+} |
+ |
+static int fts5ConfigParseColumn( |
+ Fts5Config *p, |
+ char *zCol, |
+ char *zArg, |
+ char **pzErr |
+){ |
+ int rc = SQLITE_OK; |
+ if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) |
+ || 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME) |
+ ){ |
+ *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol); |
+ rc = SQLITE_ERROR; |
+ }else if( zArg ){ |
+ if( 0==sqlite3_stricmp(zArg, "unindexed") ){ |
+ p->abUnindexed[p->nCol] = 1; |
+ }else{ |
+ *pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg); |
+ rc = SQLITE_ERROR; |
+ } |
+ } |
+ |
+ p->azCol[p->nCol++] = zCol; |
+ return rc; |
+} |
+ |
+/* |
+** Populate the Fts5Config.zContentExprlist string. |
+*/ |
+static int fts5ConfigMakeExprlist(Fts5Config *p){ |
+ int i; |
+ int rc = SQLITE_OK; |
+ Fts5Buffer buf = {0, 0, 0}; |
+ |
+ sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid); |
+ if( p->eContent!=FTS5_CONTENT_NONE ){ |
+ for(i=0; i<p->nCol; i++){ |
+ if( p->eContent==FTS5_CONTENT_EXTERNAL ){ |
+ sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]); |
+ }else{ |
+ sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i); |
+ } |
+ } |
+ } |
+ |
+ assert( p->zContentExprlist==0 ); |
+ p->zContentExprlist = (char*)buf.p; |
+ return rc; |
+} |
+ |
+/* |
+** Arguments nArg/azArg contain the string arguments passed to the xCreate |
+** or xConnect method of the virtual table. This function attempts to |
+** allocate an instance of Fts5Config containing the results of parsing |
+** those arguments. |
+** |
+** If successful, SQLITE_OK is returned and *ppOut is set to point to the |
+** new Fts5Config object. If an error occurs, an SQLite error code is |
+** returned, *ppOut is set to NULL and an error message may be left in |
+** *pzErr. It is the responsibility of the caller to eventually free any |
+** such error message using sqlite3_free(). |
+*/ |
+static int sqlite3Fts5ConfigParse( |
+ Fts5Global *pGlobal, |
+ sqlite3 *db, |
+ int nArg, /* Number of arguments */ |
+ const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */ |
+ Fts5Config **ppOut, /* OUT: Results of parse */ |
+ char **pzErr /* OUT: Error message */ |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ Fts5Config *pRet; /* New object to return */ |
+ int i; |
+ int nByte; |
+ |
+ *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); |
+ if( pRet==0 ) return SQLITE_NOMEM; |
+ memset(pRet, 0, sizeof(Fts5Config)); |
+ pRet->db = db; |
+ pRet->iCookie = -1; |
+ |
+ nByte = nArg * (sizeof(char*) + sizeof(u8)); |
+ pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); |
+ pRet->abUnindexed = (u8*)&pRet->azCol[nArg]; |
+ pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); |
+ pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); |
+ pRet->bColumnsize = 1; |
+ pRet->eDetail = FTS5_DETAIL_FULL; |
+#ifdef SQLITE_DEBUG |
+ pRet->bPrefixIndex = 1; |
+#endif |
+ if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ |
+ *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); |
+ rc = SQLITE_ERROR; |
+ } |
+ |
+ for(i=3; rc==SQLITE_OK && i<nArg; i++){ |
+ const char *zOrig = azArg[i]; |
+ const char *z; |
+ char *zOne = 0; |
+ char *zTwo = 0; |
+ int bOption = 0; |
+ int bMustBeCol = 0; |
+ |
+ z = fts5ConfigGobbleWord(&rc, zOrig, &zOne, &bMustBeCol); |
+ z = fts5ConfigSkipWhitespace(z); |
+ if( z && *z=='=' ){ |
+ bOption = 1; |
+ z++; |
+ if( bMustBeCol ) z = 0; |
+ } |
+ z = fts5ConfigSkipWhitespace(z); |
+ if( z && z[0] ){ |
+ int bDummy; |
+ z = fts5ConfigGobbleWord(&rc, z, &zTwo, &bDummy); |
+ if( z && z[0] ) z = 0; |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ if( z==0 ){ |
+ *pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ if( bOption ){ |
+ rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr); |
+ }else{ |
+ rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr); |
+ zOne = 0; |
+ } |
+ } |
+ } |
+ |
+ sqlite3_free(zOne); |
+ sqlite3_free(zTwo); |
+ } |
+ |
+ /* If a tokenizer= option was successfully parsed, the tokenizer has |
+ ** already been allocated. Otherwise, allocate an instance of the default |
+ ** tokenizer (unicode61) now. */ |
+ if( rc==SQLITE_OK && pRet->pTok==0 ){ |
+ rc = fts5ConfigDefaultTokenizer(pGlobal, pRet); |
+ } |
+ |
+ /* If no zContent option was specified, fill in the default values. */ |
+ if( rc==SQLITE_OK && pRet->zContent==0 ){ |
+ const char *zTail = 0; |
+ assert( pRet->eContent==FTS5_CONTENT_NORMAL |
+ || pRet->eContent==FTS5_CONTENT_NONE |
+ ); |
+ if( pRet->eContent==FTS5_CONTENT_NORMAL ){ |
+ zTail = "content"; |
+ }else if( pRet->bColumnsize ){ |
+ zTail = "docsize"; |
+ } |
+ |
+ if( zTail ){ |
+ pRet->zContent = sqlite3Fts5Mprintf( |
+ &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail |
+ ); |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK && pRet->zContentRowid==0 ){ |
+ pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1); |
+ } |
+ |
+ /* Formulate the zContentExprlist text */ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5ConfigMakeExprlist(pRet); |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3Fts5ConfigFree(pRet); |
+ *ppOut = 0; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Free the configuration object passed as the only argument. |
+*/ |
+static void sqlite3Fts5ConfigFree(Fts5Config *pConfig){ |
+ if( pConfig ){ |
+ int i; |
+ if( pConfig->pTok ){ |
+ pConfig->pTokApi->xDelete(pConfig->pTok); |
+ } |
+ sqlite3_free(pConfig->zDb); |
+ sqlite3_free(pConfig->zName); |
+ for(i=0; i<pConfig->nCol; i++){ |
+ sqlite3_free(pConfig->azCol[i]); |
+ } |
+ sqlite3_free(pConfig->azCol); |
+ sqlite3_free(pConfig->aPrefix); |
+ sqlite3_free(pConfig->zRank); |
+ sqlite3_free(pConfig->zRankArgs); |
+ sqlite3_free(pConfig->zContent); |
+ sqlite3_free(pConfig->zContentRowid); |
+ sqlite3_free(pConfig->zContentExprlist); |
+ sqlite3_free(pConfig); |
+ } |
+} |
+ |
+/* |
+** Call sqlite3_declare_vtab() based on the contents of the configuration |
+** object passed as the only argument. Return SQLITE_OK if successful, or |
+** an SQLite error code if an error occurs. |
+*/ |
+static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){ |
+ int i; |
+ int rc = SQLITE_OK; |
+ char *zSql; |
+ |
+ zSql = sqlite3Fts5Mprintf(&rc, "CREATE TABLE x("); |
+ for(i=0; zSql && i<pConfig->nCol; i++){ |
+ const char *zSep = (i==0?"":", "); |
+ zSql = sqlite3Fts5Mprintf(&rc, "%z%s%Q", zSql, zSep, pConfig->azCol[i]); |
+ } |
+ zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)", |
+ zSql, pConfig->zName, FTS5_RANK_NAME |
+ ); |
+ |
+ assert( zSql || rc==SQLITE_NOMEM ); |
+ if( zSql ){ |
+ rc = sqlite3_declare_vtab(pConfig->db, zSql); |
+ sqlite3_free(zSql); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Tokenize the text passed via the second and third arguments. |
+** |
+** The callback is invoked once for each token in the input text. The |
+** arguments passed to it are, in order: |
+** |
+** void *pCtx // Copy of 4th argument to sqlite3Fts5Tokenize() |
+** const char *pToken // Pointer to buffer containing token |
+** int nToken // Size of token in bytes |
+** int iStart // Byte offset of start of token within input text |
+** int iEnd // Byte offset of end of token within input text |
+** int iPos // Position of token in input (first token is 0) |
+** |
+** If the callback returns a non-zero value the tokenization is abandoned |
+** and no further callbacks are issued. |
+** |
+** This function returns SQLITE_OK if successful or an SQLite error code |
+** if an error occurs. If the tokenization was abandoned early because |
+** the callback returned SQLITE_DONE, this is not an error and this function |
+** still returns SQLITE_OK. Or, if the tokenization was abandoned early |
+** because the callback returned another non-zero value, it is assumed |
+** to be an SQLite error code and returned to the caller. |
+*/ |
+static int sqlite3Fts5Tokenize( |
+ Fts5Config *pConfig, /* FTS5 Configuration object */ |
+ int flags, /* FTS5_TOKENIZE_* flags */ |
+ const char *pText, int nText, /* Text to tokenize */ |
+ void *pCtx, /* Context passed to xToken() */ |
+ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ |
+){ |
+ if( pText==0 ) return SQLITE_OK; |
+ return pConfig->pTokApi->xTokenize( |
+ pConfig->pTok, pCtx, flags, pText, nText, xToken |
+ ); |
+} |
+ |
+/* |
+** Argument pIn points to the first character in what is expected to be |
+** a comma-separated list of SQL literals followed by a ')' character. |
+** If it actually is this, return a pointer to the ')'. Otherwise, return |
+** NULL to indicate a parse error. |
+*/ |
+static const char *fts5ConfigSkipArgs(const char *pIn){ |
+ const char *p = pIn; |
+ |
+ while( 1 ){ |
+ p = fts5ConfigSkipWhitespace(p); |
+ p = fts5ConfigSkipLiteral(p); |
+ p = fts5ConfigSkipWhitespace(p); |
+ if( p==0 || *p==')' ) break; |
+ if( *p!=',' ){ |
+ p = 0; |
+ break; |
+ } |
+ p++; |
+ } |
+ |
+ return p; |
+} |
+ |
+/* |
+** Parameter zIn contains a rank() function specification. The format of |
+** this is: |
+** |
+** + Bareword (function name) |
+** + Open parenthesis - "(" |
+** + Zero or more SQL literals in a comma separated list |
+** + Close parenthesis - ")" |
+*/ |
+static int sqlite3Fts5ConfigParseRank( |
+ const char *zIn, /* Input string */ |
+ char **pzRank, /* OUT: Rank function name */ |
+ char **pzRankArgs /* OUT: Rank function arguments */ |
+){ |
+ const char *p = zIn; |
+ const char *pRank; |
+ char *zRank = 0; |
+ char *zRankArgs = 0; |
+ int rc = SQLITE_OK; |
+ |
+ *pzRank = 0; |
+ *pzRankArgs = 0; |
+ |
+ if( p==0 ){ |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ p = fts5ConfigSkipWhitespace(p); |
+ pRank = p; |
+ p = fts5ConfigSkipBareword(p); |
+ |
+ if( p ){ |
+ zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank); |
+ if( zRank ) memcpy(zRank, pRank, p-pRank); |
+ }else{ |
+ rc = SQLITE_ERROR; |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ p = fts5ConfigSkipWhitespace(p); |
+ if( *p!='(' ) rc = SQLITE_ERROR; |
+ p++; |
+ } |
+ if( rc==SQLITE_OK ){ |
+ const char *pArgs; |
+ p = fts5ConfigSkipWhitespace(p); |
+ pArgs = p; |
+ if( *p!=')' ){ |
+ p = fts5ConfigSkipArgs(p); |
+ if( p==0 ){ |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs); |
+ if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs); |
+ } |
+ } |
+ } |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_free(zRank); |
+ assert( zRankArgs==0 ); |
+ }else{ |
+ *pzRank = zRank; |
+ *pzRankArgs = zRankArgs; |
+ } |
+ return rc; |
+} |
+ |
+static int sqlite3Fts5ConfigSetValue( |
+ Fts5Config *pConfig, |
+ const char *zKey, |
+ sqlite3_value *pVal, |
+ int *pbBadkey |
+){ |
+ int rc = SQLITE_OK; |
+ |
+ if( 0==sqlite3_stricmp(zKey, "pgsz") ){ |
+ int pgsz = 0; |
+ if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ |
+ pgsz = sqlite3_value_int(pVal); |
+ } |
+ if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){ |
+ *pbBadkey = 1; |
+ }else{ |
+ pConfig->pgsz = pgsz; |
+ } |
+ } |
+ |
+ else if( 0==sqlite3_stricmp(zKey, "hashsize") ){ |
+ int nHashSize = -1; |
+ if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ |
+ nHashSize = sqlite3_value_int(pVal); |
+ } |
+ if( nHashSize<=0 ){ |
+ *pbBadkey = 1; |
+ }else{ |
+ pConfig->nHashSize = nHashSize; |
+ } |
+ } |
+ |
+ else if( 0==sqlite3_stricmp(zKey, "automerge") ){ |
+ int nAutomerge = -1; |
+ if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ |
+ nAutomerge = sqlite3_value_int(pVal); |
+ } |
+ if( nAutomerge<0 || nAutomerge>64 ){ |
+ *pbBadkey = 1; |
+ }else{ |
+ if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE; |
+ pConfig->nAutomerge = nAutomerge; |
+ } |
+ } |
+ |
+ else if( 0==sqlite3_stricmp(zKey, "usermerge") ){ |
+ int nUsermerge = -1; |
+ if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ |
+ nUsermerge = sqlite3_value_int(pVal); |
+ } |
+ if( nUsermerge<2 || nUsermerge>16 ){ |
+ *pbBadkey = 1; |
+ }else{ |
+ pConfig->nUsermerge = nUsermerge; |
+ } |
+ } |
+ |
+ else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){ |
+ int nCrisisMerge = -1; |
+ if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ |
+ nCrisisMerge = sqlite3_value_int(pVal); |
+ } |
+ if( nCrisisMerge<0 ){ |
+ *pbBadkey = 1; |
+ }else{ |
+ if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; |
+ pConfig->nCrisisMerge = nCrisisMerge; |
+ } |
+ } |
+ |
+ else if( 0==sqlite3_stricmp(zKey, "rank") ){ |
+ const char *zIn = (const char*)sqlite3_value_text(pVal); |
+ char *zRank; |
+ char *zRankArgs; |
+ rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_free(pConfig->zRank); |
+ sqlite3_free(pConfig->zRankArgs); |
+ pConfig->zRank = zRank; |
+ pConfig->zRankArgs = zRankArgs; |
+ }else if( rc==SQLITE_ERROR ){ |
+ rc = SQLITE_OK; |
+ *pbBadkey = 1; |
+ } |
+ }else{ |
+ *pbBadkey = 1; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Load the contents of the %_config table into memory. |
+*/ |
+static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ |
+ const char *zSelect = "SELECT k, v FROM %Q.'%q_config'"; |
+ char *zSql; |
+ sqlite3_stmt *p = 0; |
+ int rc = SQLITE_OK; |
+ int iVersion = 0; |
+ |
+ /* Set default values */ |
+ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; |
+ pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; |
+ pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; |
+ pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; |
+ pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; |
+ |
+ zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); |
+ if( zSql ){ |
+ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); |
+ sqlite3_free(zSql); |
+ } |
+ |
+ assert( rc==SQLITE_OK || p==0 ); |
+ if( rc==SQLITE_OK ){ |
+ while( SQLITE_ROW==sqlite3_step(p) ){ |
+ const char *zK = (const char*)sqlite3_column_text(p, 0); |
+ sqlite3_value *pVal = sqlite3_column_value(p, 1); |
+ if( 0==sqlite3_stricmp(zK, "version") ){ |
+ iVersion = sqlite3_value_int(pVal); |
+ }else{ |
+ int bDummy = 0; |
+ sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, &bDummy); |
+ } |
+ } |
+ rc = sqlite3_finalize(p); |
+ } |
+ |
+ if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){ |
+ rc = SQLITE_ERROR; |
+ if( pConfig->pzErrmsg ){ |
+ assert( 0==*pConfig->pzErrmsg ); |
+ *pConfig->pzErrmsg = sqlite3_mprintf( |
+ "invalid fts5 file format (found %d, expected %d) - run 'rebuild'", |
+ iVersion, FTS5_CURRENT_VERSION |
+ ); |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ pConfig->iCookie = iCookie; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** 2014 May 31 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+*/ |
+ |
+ |
+ |
+/* #include "fts5Int.h" */ |
+/* #include "fts5parse.h" */ |
+ |
+/* |
+** All token types in the generated fts5parse.h file are greater than 0. |
+*/ |
+#define FTS5_EOF 0 |
+ |
+#define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) |
+ |
+typedef struct Fts5ExprTerm Fts5ExprTerm; |
+ |
+/* |
+** Functions generated by lemon from fts5parse.y. |
+*/ |
+static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(u64)); |
+static void sqlite3Fts5ParserFree(void*, void (*freeProc)(void*)); |
+static void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*); |
+#ifndef NDEBUG |
+/* #include <stdio.h> */ |
+static void sqlite3Fts5ParserTrace(FILE*, char*); |
+#endif |
+ |
+ |
+struct Fts5Expr { |
+ Fts5Index *pIndex; |
+ Fts5Config *pConfig; |
+ Fts5ExprNode *pRoot; |
+ int bDesc; /* Iterate in descending rowid order */ |
+ int nPhrase; /* Number of phrases in expression */ |
+ Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ |
+}; |
+ |
+/* |
+** eType: |
+** Expression node type. Always one of: |
+** |
+** FTS5_AND (nChild, apChild valid) |
+** FTS5_OR (nChild, apChild valid) |
+** FTS5_NOT (nChild, apChild valid) |
+** FTS5_STRING (pNear valid) |
+** FTS5_TERM (pNear valid) |
+*/ |
+struct Fts5ExprNode { |
+ int eType; /* Node type */ |
+ int bEof; /* True at EOF */ |
+ int bNomatch; /* True if entry is not a match */ |
+ |
+ /* Next method for this node. */ |
+ int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64); |
+ |
+ i64 iRowid; /* Current rowid */ |
+ Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */ |
+ |
+ /* Child nodes. For a NOT node, this array always contains 2 entries. For |
+ ** AND or OR nodes, it contains 2 or more entries. */ |
+ int nChild; /* Number of child nodes */ |
+ Fts5ExprNode *apChild[1]; /* Array of child nodes */ |
+}; |
+ |
+#define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING) |
+ |
+/* |
+** Invoke the xNext method of an Fts5ExprNode object. This macro should be |
+** used as if it has the same signature as the xNext() methods themselves. |
+*/ |
+#define fts5ExprNodeNext(a,b,c,d) (b)->xNext((a), (b), (c), (d)) |
+ |
+/* |
+** An instance of the following structure represents a single search term |
+** or term prefix. |
+*/ |
+struct Fts5ExprTerm { |
+ int bPrefix; /* True for a prefix term */ |
+ char *zTerm; /* nul-terminated term */ |
+ Fts5IndexIter *pIter; /* Iterator for this term */ |
+ Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ |
+}; |
+ |
+/* |
+** A phrase. One or more terms that must appear in a contiguous sequence |
+** within a document for it to match. |
+*/ |
+struct Fts5ExprPhrase { |
+ Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */ |
+ Fts5Buffer poslist; /* Current position list */ |
+ int nTerm; /* Number of entries in aTerm[] */ |
+ Fts5ExprTerm aTerm[1]; /* Terms that make up this phrase */ |
+}; |
+ |
+/* |
+** One or more phrases that must appear within a certain token distance of |
+** each other within each matching document. |
+*/ |
+struct Fts5ExprNearset { |
+ int nNear; /* NEAR parameter */ |
+ Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */ |
+ int nPhrase; /* Number of entries in aPhrase[] array */ |
+ Fts5ExprPhrase *apPhrase[1]; /* Array of phrase pointers */ |
+}; |
+ |
+ |
+/* |
+** Parse context. |
+*/ |
+struct Fts5Parse { |
+ Fts5Config *pConfig; |
+ char *zErr; |
+ int rc; |
+ int nPhrase; /* Size of apPhrase array */ |
+ Fts5ExprPhrase **apPhrase; /* Array of all phrases */ |
+ Fts5ExprNode *pExpr; /* Result of a successful parse */ |
+}; |
+ |
+static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){ |
+ va_list ap; |
+ va_start(ap, zFmt); |
+ if( pParse->rc==SQLITE_OK ){ |
+ pParse->zErr = sqlite3_vmprintf(zFmt, ap); |
+ pParse->rc = SQLITE_ERROR; |
+ } |
+ va_end(ap); |
+} |
+ |
+static int fts5ExprIsspace(char t){ |
+ return t==' ' || t=='\t' || t=='\n' || t=='\r'; |
+} |
+ |
+/* |
+** Read the first token from the nul-terminated string at *pz. |
+*/ |
+static int fts5ExprGetToken( |
+ Fts5Parse *pParse, |
+ const char **pz, /* IN/OUT: Pointer into buffer */ |
+ Fts5Token *pToken |
+){ |
+ const char *z = *pz; |
+ int tok; |
+ |
+ /* Skip past any whitespace */ |
+ while( fts5ExprIsspace(*z) ) z++; |
+ |
+ pToken->p = z; |
+ pToken->n = 1; |
+ switch( *z ){ |
+ case '(': tok = FTS5_LP; break; |
+ case ')': tok = FTS5_RP; break; |
+ case '{': tok = FTS5_LCP; break; |
+ case '}': tok = FTS5_RCP; break; |
+ case ':': tok = FTS5_COLON; break; |
+ case ',': tok = FTS5_COMMA; break; |
+ case '+': tok = FTS5_PLUS; break; |
+ case '*': tok = FTS5_STAR; break; |
+ case '-': tok = FTS5_MINUS; break; |
+ case '\0': tok = FTS5_EOF; break; |
+ |
+ case '"': { |
+ const char *z2; |
+ tok = FTS5_STRING; |
+ |
+ for(z2=&z[1]; 1; z2++){ |
+ if( z2[0]=='"' ){ |
+ z2++; |
+ if( z2[0]!='"' ) break; |
+ } |
+ if( z2[0]=='\0' ){ |
+ sqlite3Fts5ParseError(pParse, "unterminated string"); |
+ return FTS5_EOF; |
+ } |
+ } |
+ pToken->n = (z2 - z); |
+ break; |
+ } |
+ |
+ default: { |
+ const char *z2; |
+ if( sqlite3Fts5IsBareword(z[0])==0 ){ |
+ sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z); |
+ return FTS5_EOF; |
+ } |
+ tok = FTS5_STRING; |
+ for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++); |
+ pToken->n = (z2 - z); |
+ if( pToken->n==2 && memcmp(pToken->p, "OR", 2)==0 ) tok = FTS5_OR; |
+ if( pToken->n==3 && memcmp(pToken->p, "NOT", 3)==0 ) tok = FTS5_NOT; |
+ if( pToken->n==3 && memcmp(pToken->p, "AND", 3)==0 ) tok = FTS5_AND; |
+ break; |
+ } |
+ } |
+ |
+ *pz = &pToken->p[pToken->n]; |
+ return tok; |
+} |
+ |
+static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc((int)t); } |
+static void fts5ParseFree(void *p){ sqlite3_free(p); } |
+ |
+static int sqlite3Fts5ExprNew( |
+ Fts5Config *pConfig, /* FTS5 Configuration */ |
+ const char *zExpr, /* Expression text */ |
+ Fts5Expr **ppNew, |
+ char **pzErr |
+){ |
+ Fts5Parse sParse; |
+ Fts5Token token; |
+ const char *z = zExpr; |
+ int t; /* Next token type */ |
+ void *pEngine; |
+ Fts5Expr *pNew; |
+ |
+ *ppNew = 0; |
+ *pzErr = 0; |
+ memset(&sParse, 0, sizeof(sParse)); |
+ pEngine = sqlite3Fts5ParserAlloc(fts5ParseAlloc); |
+ if( pEngine==0 ){ return SQLITE_NOMEM; } |
+ sParse.pConfig = pConfig; |
+ |
+ do { |
+ t = fts5ExprGetToken(&sParse, &z, &token); |
+ sqlite3Fts5Parser(pEngine, t, token, &sParse); |
+ }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); |
+ sqlite3Fts5ParserFree(pEngine, fts5ParseFree); |
+ |
+ assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); |
+ if( sParse.rc==SQLITE_OK ){ |
+ *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); |
+ if( pNew==0 ){ |
+ sParse.rc = SQLITE_NOMEM; |
+ sqlite3Fts5ParseNodeFree(sParse.pExpr); |
+ }else{ |
+ if( !sParse.pExpr ){ |
+ const int nByte = sizeof(Fts5ExprNode); |
+ pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte); |
+ if( pNew->pRoot ){ |
+ pNew->pRoot->bEof = 1; |
+ } |
+ }else{ |
+ pNew->pRoot = sParse.pExpr; |
+ } |
+ pNew->pIndex = 0; |
+ pNew->pConfig = pConfig; |
+ pNew->apExprPhrase = sParse.apPhrase; |
+ pNew->nPhrase = sParse.nPhrase; |
+ sParse.apPhrase = 0; |
+ } |
+ }else{ |
+ sqlite3Fts5ParseNodeFree(sParse.pExpr); |
+ } |
+ |
+ sqlite3_free(sParse.apPhrase); |
+ *pzErr = sParse.zErr; |
+ return sParse.rc; |
+} |
+ |
+/* |
+** Free the expression node object passed as the only argument. |
+*/ |
+static void sqlite3Fts5ParseNodeFree(Fts5ExprNode *p){ |
+ if( p ){ |
+ int i; |
+ for(i=0; i<p->nChild; i++){ |
+ sqlite3Fts5ParseNodeFree(p->apChild[i]); |
+ } |
+ sqlite3Fts5ParseNearsetFree(p->pNear); |
+ sqlite3_free(p); |
+ } |
+} |
+ |
+/* |
+** Free the expression object passed as the only argument. |
+*/ |
+static void sqlite3Fts5ExprFree(Fts5Expr *p){ |
+ if( p ){ |
+ sqlite3Fts5ParseNodeFree(p->pRoot); |
+ sqlite3_free(p->apExprPhrase); |
+ sqlite3_free(p); |
+ } |
+} |
+ |
+/* |
+** Argument pTerm must be a synonym iterator. Return the current rowid |
+** that it points to. |
+*/ |
+static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ |
+ i64 iRet = 0; |
+ int bRetValid = 0; |
+ Fts5ExprTerm *p; |
+ |
+ assert( pTerm->pSynonym ); |
+ assert( bDesc==0 || bDesc==1 ); |
+ for(p=pTerm; p; p=p->pSynonym){ |
+ if( 0==sqlite3Fts5IterEof(p->pIter) ){ |
+ i64 iRowid = p->pIter->iRowid; |
+ if( bRetValid==0 || (bDesc!=(iRowid<iRet)) ){ |
+ iRet = iRowid; |
+ bRetValid = 1; |
+ } |
+ } |
+ } |
+ |
+ if( pbEof && bRetValid==0 ) *pbEof = 1; |
+ return iRet; |
+} |
+ |
+/* |
+** Argument pTerm must be a synonym iterator. |
+*/ |
+static int fts5ExprSynonymList( |
+ Fts5ExprTerm *pTerm, |
+ i64 iRowid, |
+ Fts5Buffer *pBuf, /* Use this buffer for space if required */ |
+ u8 **pa, int *pn |
+){ |
+ Fts5PoslistReader aStatic[4]; |
+ Fts5PoslistReader *aIter = aStatic; |
+ int nIter = 0; |
+ int nAlloc = 4; |
+ int rc = SQLITE_OK; |
+ Fts5ExprTerm *p; |
+ |
+ assert( pTerm->pSynonym ); |
+ for(p=pTerm; p; p=p->pSynonym){ |
+ Fts5IndexIter *pIter = p->pIter; |
+ if( sqlite3Fts5IterEof(pIter)==0 && pIter->iRowid==iRowid ){ |
+ if( pIter->nData==0 ) continue; |
+ if( nIter==nAlloc ){ |
+ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; |
+ Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); |
+ if( aNew==0 ){ |
+ rc = SQLITE_NOMEM; |
+ goto synonym_poslist_out; |
+ } |
+ memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); |
+ nAlloc = nAlloc*2; |
+ if( aIter!=aStatic ) sqlite3_free(aIter); |
+ aIter = aNew; |
+ } |
+ sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]); |
+ assert( aIter[nIter].bEof==0 ); |
+ nIter++; |
+ } |
+ } |
+ |
+ if( nIter==1 ){ |
+ *pa = (u8*)aIter[0].a; |
+ *pn = aIter[0].n; |
+ }else{ |
+ Fts5PoslistWriter writer = {0}; |
+ i64 iPrev = -1; |
+ fts5BufferZero(pBuf); |
+ while( 1 ){ |
+ int i; |
+ i64 iMin = FTS5_LARGEST_INT64; |
+ for(i=0; i<nIter; i++){ |
+ if( aIter[i].bEof==0 ){ |
+ if( aIter[i].iPos==iPrev ){ |
+ if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) continue; |
+ } |
+ if( aIter[i].iPos<iMin ){ |
+ iMin = aIter[i].iPos; |
+ } |
+ } |
+ } |
+ if( iMin==FTS5_LARGEST_INT64 || rc!=SQLITE_OK ) break; |
+ rc = sqlite3Fts5PoslistWriterAppend(pBuf, &writer, iMin); |
+ iPrev = iMin; |
+ } |
+ if( rc==SQLITE_OK ){ |
+ *pa = pBuf->p; |
+ *pn = pBuf->n; |
+ } |
+ } |
+ |
+ synonym_poslist_out: |
+ if( aIter!=aStatic ) sqlite3_free(aIter); |
+ return rc; |
+} |
+ |
+ |
+/* |
+** All individual term iterators in pPhrase are guaranteed to be valid and |
+** pointing to the same rowid when this function is called. This function |
+** checks if the current rowid really is a match, and if so populates |
+** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch |
+** is set to true if this is really a match, or false otherwise. |
+** |
+** SQLITE_OK is returned if an error occurs, or an SQLite error code |
+** otherwise. It is not considered an error code if the current rowid is |
+** not a match. |
+*/ |
+static int fts5ExprPhraseIsMatch( |
+ Fts5ExprNode *pNode, /* Node pPhrase belongs to */ |
+ Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ |
+ int *pbMatch /* OUT: Set to true if really a match */ |
+){ |
+ Fts5PoslistWriter writer = {0}; |
+ Fts5PoslistReader aStatic[4]; |
+ Fts5PoslistReader *aIter = aStatic; |
+ int i; |
+ int rc = SQLITE_OK; |
+ |
+ fts5BufferZero(&pPhrase->poslist); |
+ |
+ /* If the aStatic[] array is not large enough, allocate a large array |
+ ** using sqlite3_malloc(). This approach could be improved upon. */ |
+ if( pPhrase->nTerm>ArraySize(aStatic) ){ |
+ int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm; |
+ aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte); |
+ if( !aIter ) return SQLITE_NOMEM; |
+ } |
+ memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm); |
+ |
+ /* Initialize a term iterator for each term in the phrase */ |
+ for(i=0; i<pPhrase->nTerm; i++){ |
+ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; |
+ int n = 0; |
+ int bFlag = 0; |
+ u8 *a = 0; |
+ if( pTerm->pSynonym ){ |
+ Fts5Buffer buf = {0, 0, 0}; |
+ rc = fts5ExprSynonymList(pTerm, pNode->iRowid, &buf, &a, &n); |
+ if( rc ){ |
+ sqlite3_free(a); |
+ goto ismatch_out; |
+ } |
+ if( a==buf.p ) bFlag = 1; |
+ }else{ |
+ a = (u8*)pTerm->pIter->pData; |
+ n = pTerm->pIter->nData; |
+ } |
+ sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); |
+ aIter[i].bFlag = (u8)bFlag; |
+ if( aIter[i].bEof ) goto ismatch_out; |
+ } |
+ |
+ while( 1 ){ |
+ int bMatch; |
+ i64 iPos = aIter[0].iPos; |
+ do { |
+ bMatch = 1; |
+ for(i=0; i<pPhrase->nTerm; i++){ |
+ Fts5PoslistReader *pPos = &aIter[i]; |
+ i64 iAdj = iPos + i; |
+ if( pPos->iPos!=iAdj ){ |
+ bMatch = 0; |
+ while( pPos->iPos<iAdj ){ |
+ if( sqlite3Fts5PoslistReaderNext(pPos) ) goto ismatch_out; |
+ } |
+ if( pPos->iPos>iAdj ) iPos = pPos->iPos-i; |
+ } |
+ } |
+ }while( bMatch==0 ); |
+ |
+ /* Append position iPos to the output */ |
+ rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); |
+ if( rc!=SQLITE_OK ) goto ismatch_out; |
+ |
+ for(i=0; i<pPhrase->nTerm; i++){ |
+ if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out; |
+ } |
+ } |
+ |
+ ismatch_out: |
+ *pbMatch = (pPhrase->poslist.n>0); |
+ for(i=0; i<pPhrase->nTerm; i++){ |
+ if( aIter[i].bFlag ) sqlite3_free((u8*)aIter[i].a); |
+ } |
+ if( aIter!=aStatic ) sqlite3_free(aIter); |
+ return rc; |
+} |
+ |
+typedef struct Fts5LookaheadReader Fts5LookaheadReader; |
+struct Fts5LookaheadReader { |
+ const u8 *a; /* Buffer containing position list */ |
+ int n; /* Size of buffer a[] in bytes */ |
+ int i; /* Current offset in position list */ |
+ i64 iPos; /* Current position */ |
+ i64 iLookahead; /* Next position */ |
+}; |
+ |
+#define FTS5_LOOKAHEAD_EOF (((i64)1) << 62) |
+ |
+static int fts5LookaheadReaderNext(Fts5LookaheadReader *p){ |
+ p->iPos = p->iLookahead; |
+ if( sqlite3Fts5PoslistNext64(p->a, p->n, &p->i, &p->iLookahead) ){ |
+ p->iLookahead = FTS5_LOOKAHEAD_EOF; |
+ } |
+ return (p->iPos==FTS5_LOOKAHEAD_EOF); |
+} |
+ |
+static int fts5LookaheadReaderInit( |
+ const u8 *a, int n, /* Buffer to read position list from */ |
+ Fts5LookaheadReader *p /* Iterator object to initialize */ |
+){ |
+ memset(p, 0, sizeof(Fts5LookaheadReader)); |
+ p->a = a; |
+ p->n = n; |
+ fts5LookaheadReaderNext(p); |
+ return fts5LookaheadReaderNext(p); |
+} |
+ |
+typedef struct Fts5NearTrimmer Fts5NearTrimmer; |
+struct Fts5NearTrimmer { |
+ Fts5LookaheadReader reader; /* Input iterator */ |
+ Fts5PoslistWriter writer; /* Writer context */ |
+ Fts5Buffer *pOut; /* Output poslist */ |
+}; |
+ |
+/* |
+** The near-set object passed as the first argument contains more than |
+** one phrase. All phrases currently point to the same row. The |
+** Fts5ExprPhrase.poslist buffers are populated accordingly. This function |
+** tests if the current row contains instances of each phrase sufficiently |
+** close together to meet the NEAR constraint. Non-zero is returned if it |
+** does, or zero otherwise. |
+** |
+** If in/out parameter (*pRc) is set to other than SQLITE_OK when this |
+** function is called, it is a no-op. Or, if an error (e.g. SQLITE_NOMEM) |
+** occurs within this function (*pRc) is set accordingly before returning. |
+** The return value is undefined in both these cases. |
+** |
+** If no error occurs and non-zero (a match) is returned, the position-list |
+** of each phrase object is edited to contain only those entries that |
+** meet the constraint before returning. |
+*/ |
+static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){ |
+ Fts5NearTrimmer aStatic[4]; |
+ Fts5NearTrimmer *a = aStatic; |
+ Fts5ExprPhrase **apPhrase = pNear->apPhrase; |
+ |
+ int i; |
+ int rc = *pRc; |
+ int bMatch; |
+ |
+ assert( pNear->nPhrase>1 ); |
+ |
+ /* If the aStatic[] array is not large enough, allocate a large array |
+ ** using sqlite3_malloc(). This approach could be improved upon. */ |
+ if( pNear->nPhrase>ArraySize(aStatic) ){ |
+ int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase; |
+ a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte); |
+ }else{ |
+ memset(aStatic, 0, sizeof(aStatic)); |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ *pRc = rc; |
+ return 0; |
+ } |
+ |
+ /* Initialize a lookahead iterator for each phrase. After passing the |
+ ** buffer and buffer size to the lookaside-reader init function, zero |
+ ** the phrase poslist buffer. The new poslist for the phrase (containing |
+ ** the same entries as the original with some entries removed on account |
+ ** of the NEAR constraint) is written over the original even as it is |
+ ** being read. This is safe as the entries for the new poslist are a |
+ ** subset of the old, so it is not possible for data yet to be read to |
+ ** be overwritten. */ |
+ for(i=0; i<pNear->nPhrase; i++){ |
+ Fts5Buffer *pPoslist = &apPhrase[i]->poslist; |
+ fts5LookaheadReaderInit(pPoslist->p, pPoslist->n, &a[i].reader); |
+ pPoslist->n = 0; |
+ a[i].pOut = pPoslist; |
+ } |
+ |
+ while( 1 ){ |
+ int iAdv; |
+ i64 iMin; |
+ i64 iMax; |
+ |
+ /* This block advances the phrase iterators until they point to a set of |
+ ** entries that together comprise a match. */ |
+ iMax = a[0].reader.iPos; |
+ do { |
+ bMatch = 1; |
+ for(i=0; i<pNear->nPhrase; i++){ |
+ Fts5LookaheadReader *pPos = &a[i].reader; |
+ iMin = iMax - pNear->apPhrase[i]->nTerm - pNear->nNear; |
+ if( pPos->iPos<iMin || pPos->iPos>iMax ){ |
+ bMatch = 0; |
+ while( pPos->iPos<iMin ){ |
+ if( fts5LookaheadReaderNext(pPos) ) goto ismatch_out; |
+ } |
+ if( pPos->iPos>iMax ) iMax = pPos->iPos; |
+ } |
+ } |
+ }while( bMatch==0 ); |
+ |
+ /* Add an entry to each output position list */ |
+ for(i=0; i<pNear->nPhrase; i++){ |
+ i64 iPos = a[i].reader.iPos; |
+ Fts5PoslistWriter *pWriter = &a[i].writer; |
+ if( a[i].pOut->n==0 || iPos!=pWriter->iPrev ){ |
+ sqlite3Fts5PoslistWriterAppend(a[i].pOut, pWriter, iPos); |
+ } |
+ } |
+ |
+ iAdv = 0; |
+ iMin = a[0].reader.iLookahead; |
+ for(i=0; i<pNear->nPhrase; i++){ |
+ if( a[i].reader.iLookahead < iMin ){ |
+ iMin = a[i].reader.iLookahead; |
+ iAdv = i; |
+ } |
+ } |
+ if( fts5LookaheadReaderNext(&a[iAdv].reader) ) goto ismatch_out; |
+ } |
+ |
+ ismatch_out: { |
+ int bRet = a[0].pOut->n>0; |
+ *pRc = rc; |
+ if( a!=aStatic ) sqlite3_free(a); |
+ return bRet; |
+ } |
+} |
+ |
+/* |
+** Advance iterator pIter until it points to a value equal to or laster |
+** than the initial value of *piLast. If this means the iterator points |
+** to a value laster than *piLast, update *piLast to the new lastest value. |
+** |
+** If the iterator reaches EOF, set *pbEof to true before returning. If |
+** an error occurs, set *pRc to an error code. If either *pbEof or *pRc |
+** are set, return a non-zero value. Otherwise, return zero. |
+*/ |
+static int fts5ExprAdvanceto( |
+ Fts5IndexIter *pIter, /* Iterator to advance */ |
+ int bDesc, /* True if iterator is "rowid DESC" */ |
+ i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ |
+ int *pRc, /* OUT: Error code */ |
+ int *pbEof /* OUT: Set to true if EOF */ |
+){ |
+ i64 iLast = *piLast; |
+ i64 iRowid; |
+ |
+ iRowid = pIter->iRowid; |
+ if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){ |
+ int rc = sqlite3Fts5IterNextFrom(pIter, iLast); |
+ if( rc || sqlite3Fts5IterEof(pIter) ){ |
+ *pRc = rc; |
+ *pbEof = 1; |
+ return 1; |
+ } |
+ iRowid = pIter->iRowid; |
+ assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) ); |
+ } |
+ *piLast = iRowid; |
+ |
+ return 0; |
+} |
+ |
+static int fts5ExprSynonymAdvanceto( |
+ Fts5ExprTerm *pTerm, /* Term iterator to advance */ |
+ int bDesc, /* True if iterator is "rowid DESC" */ |
+ i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ |
+ int *pRc /* OUT: Error code */ |
+){ |
+ int rc = SQLITE_OK; |
+ i64 iLast = *piLast; |
+ Fts5ExprTerm *p; |
+ int bEof = 0; |
+ |
+ for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ |
+ if( sqlite3Fts5IterEof(p->pIter)==0 ){ |
+ i64 iRowid = p->pIter->iRowid; |
+ if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){ |
+ rc = sqlite3Fts5IterNextFrom(p->pIter, iLast); |
+ } |
+ } |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ *pRc = rc; |
+ bEof = 1; |
+ }else{ |
+ *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof); |
+ } |
+ return bEof; |
+} |
+ |
+ |
+static int fts5ExprNearTest( |
+ int *pRc, |
+ Fts5Expr *pExpr, /* Expression that pNear is a part of */ |
+ Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */ |
+){ |
+ Fts5ExprNearset *pNear = pNode->pNear; |
+ int rc = *pRc; |
+ |
+ if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){ |
+ Fts5ExprTerm *pTerm; |
+ Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; |
+ pPhrase->poslist.n = 0; |
+ for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ |
+ Fts5IndexIter *pIter = pTerm->pIter; |
+ if( sqlite3Fts5IterEof(pIter)==0 ){ |
+ if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){ |
+ pPhrase->poslist.n = 1; |
+ } |
+ } |
+ } |
+ return pPhrase->poslist.n; |
+ }else{ |
+ int i; |
+ |
+ /* Check that each phrase in the nearset matches the current row. |
+ ** Populate the pPhrase->poslist buffers at the same time. If any |
+ ** phrase is not a match, break out of the loop early. */ |
+ for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ |
+ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; |
+ if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ |
+ int bMatch = 0; |
+ rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch); |
+ if( bMatch==0 ) break; |
+ }else{ |
+ Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; |
+ fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData); |
+ } |
+ } |
+ |
+ *pRc = rc; |
+ if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ |
+ return 1; |
+ } |
+ return 0; |
+ } |
+} |
+ |
+ |
+/* |
+** Initialize all term iterators in the pNear object. If any term is found |
+** to match no documents at all, return immediately without initializing any |
+** further iterators. |
+** |
+** If an error occurs, return an SQLite error code. Otherwise, return |
+** SQLITE_OK. It is not considered an error if some term matches zero |
+** documents. |
+*/ |
+static int fts5ExprNearInitAll( |
+ Fts5Expr *pExpr, |
+ Fts5ExprNode *pNode |
+){ |
+ Fts5ExprNearset *pNear = pNode->pNear; |
+ int i; |
+ |
+ assert( pNode->bNomatch==0 ); |
+ for(i=0; i<pNear->nPhrase; i++){ |
+ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; |
+ if( pPhrase->nTerm==0 ){ |
+ pNode->bEof = 1; |
+ return SQLITE_OK; |
+ }else{ |
+ int j; |
+ for(j=0; j<pPhrase->nTerm; j++){ |
+ Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; |
+ Fts5ExprTerm *p; |
+ int bHit = 0; |
+ |
+ for(p=pTerm; p; p=p->pSynonym){ |
+ int rc; |
+ if( p->pIter ){ |
+ sqlite3Fts5IterClose(p->pIter); |
+ p->pIter = 0; |
+ } |
+ rc = sqlite3Fts5IndexQuery( |
+ pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), |
+ (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | |
+ (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), |
+ pNear->pColset, |
+ &p->pIter |
+ ); |
+ assert( (rc==SQLITE_OK)==(p->pIter!=0) ); |
+ if( rc!=SQLITE_OK ) return rc; |
+ if( 0==sqlite3Fts5IterEof(p->pIter) ){ |
+ bHit = 1; |
+ } |
+ } |
+ |
+ if( bHit==0 ){ |
+ pNode->bEof = 1; |
+ return SQLITE_OK; |
+ } |
+ } |
+ } |
+ } |
+ |
+ pNode->bEof = 0; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** If pExpr is an ASC iterator, this function returns a value with the |
+** same sign as: |
+** |
+** (iLhs - iRhs) |
+** |
+** Otherwise, if this is a DESC iterator, the opposite is returned: |
+** |
+** (iRhs - iLhs) |
+*/ |
+static int fts5RowidCmp( |
+ Fts5Expr *pExpr, |
+ i64 iLhs, |
+ i64 iRhs |
+){ |
+ assert( pExpr->bDesc==0 || pExpr->bDesc==1 ); |
+ if( pExpr->bDesc==0 ){ |
+ if( iLhs<iRhs ) return -1; |
+ return (iLhs > iRhs); |
+ }else{ |
+ if( iLhs>iRhs ) return -1; |
+ return (iLhs < iRhs); |
+ } |
+} |
+ |
+static void fts5ExprSetEof(Fts5ExprNode *pNode){ |
+ int i; |
+ pNode->bEof = 1; |
+ pNode->bNomatch = 0; |
+ for(i=0; i<pNode->nChild; i++){ |
+ fts5ExprSetEof(pNode->apChild[i]); |
+ } |
+} |
+ |
+static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){ |
+ if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ |
+ Fts5ExprNearset *pNear = pNode->pNear; |
+ int i; |
+ for(i=0; i<pNear->nPhrase; i++){ |
+ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; |
+ pPhrase->poslist.n = 0; |
+ } |
+ }else{ |
+ int i; |
+ for(i=0; i<pNode->nChild; i++){ |
+ fts5ExprNodeZeroPoslist(pNode->apChild[i]); |
+ } |
+ } |
+} |
+ |
+ |
+ |
+/* |
+** Compare the values currently indicated by the two nodes as follows: |
+** |
+** res = (*p1) - (*p2) |
+** |
+** Nodes that point to values that come later in the iteration order are |
+** considered to be larger. Nodes at EOF are the largest of all. |
+** |
+** This means that if the iteration order is ASC, then numerically larger |
+** rowids are considered larger. Or if it is the default DESC, numerically |
+** smaller rowids are larger. |
+*/ |
+static int fts5NodeCompare( |
+ Fts5Expr *pExpr, |
+ Fts5ExprNode *p1, |
+ Fts5ExprNode *p2 |
+){ |
+ if( p2->bEof ) return -1; |
+ if( p1->bEof ) return +1; |
+ return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid); |
+} |
+ |
+/* |
+** All individual term iterators in pNear are guaranteed to be valid when |
+** this function is called. This function checks if all term iterators |
+** point to the same rowid, and if not, advances them until they do. |
+** If an EOF is reached before this happens, *pbEof is set to true before |
+** returning. |
+** |
+** SQLITE_OK is returned if an error occurs, or an SQLite error code |
+** otherwise. It is not considered an error code if an iterator reaches |
+** EOF. |
+*/ |
+static int fts5ExprNodeTest_STRING( |
+ Fts5Expr *pExpr, /* Expression pPhrase belongs to */ |
+ Fts5ExprNode *pNode |
+){ |
+ Fts5ExprNearset *pNear = pNode->pNear; |
+ Fts5ExprPhrase *pLeft = pNear->apPhrase[0]; |
+ int rc = SQLITE_OK; |
+ i64 iLast; /* Lastest rowid any iterator points to */ |
+ int i, j; /* Phrase and token index, respectively */ |
+ int bMatch; /* True if all terms are at the same rowid */ |
+ const int bDesc = pExpr->bDesc; |
+ |
+ /* Check that this node should not be FTS5_TERM */ |
+ assert( pNear->nPhrase>1 |
+ || pNear->apPhrase[0]->nTerm>1 |
+ || pNear->apPhrase[0]->aTerm[0].pSynonym |
+ ); |
+ |
+ /* Initialize iLast, the "lastest" rowid any iterator points to. If the |
+ ** iterator skips through rowids in the default ascending order, this means |
+ ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it |
+ ** means the minimum rowid. */ |
+ if( pLeft->aTerm[0].pSynonym ){ |
+ iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); |
+ }else{ |
+ iLast = pLeft->aTerm[0].pIter->iRowid; |
+ } |
+ |
+ do { |
+ bMatch = 1; |
+ for(i=0; i<pNear->nPhrase; i++){ |
+ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; |
+ for(j=0; j<pPhrase->nTerm; j++){ |
+ Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; |
+ if( pTerm->pSynonym ){ |
+ i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); |
+ if( iRowid==iLast ) continue; |
+ bMatch = 0; |
+ if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ |
+ pNode->bNomatch = 0; |
+ pNode->bEof = 1; |
+ return rc; |
+ } |
+ }else{ |
+ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; |
+ if( pIter->iRowid==iLast || pIter->bEof ) continue; |
+ bMatch = 0; |
+ if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ |
+ return rc; |
+ } |
+ } |
+ } |
+ } |
+ }while( bMatch==0 ); |
+ |
+ pNode->iRowid = iLast; |
+ pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK); |
+ assert( pNode->bEof==0 || pNode->bNomatch==0 ); |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Advance the first term iterator in the first phrase of pNear. Set output |
+** variable *pbEof to true if it reaches EOF or if an error occurs. |
+** |
+** Return SQLITE_OK if successful, or an SQLite error code if an error |
+** occurs. |
+*/ |
+static int fts5ExprNodeNext_STRING( |
+ Fts5Expr *pExpr, /* Expression pPhrase belongs to */ |
+ Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */ |
+ int bFromValid, |
+ i64 iFrom |
+){ |
+ Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; |
+ int rc = SQLITE_OK; |
+ |
+ pNode->bNomatch = 0; |
+ if( pTerm->pSynonym ){ |
+ int bEof = 1; |
+ Fts5ExprTerm *p; |
+ |
+ /* Find the firstest rowid any synonym points to. */ |
+ i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); |
+ |
+ /* Advance each iterator that currently points to iRowid. Or, if iFrom |
+ ** is valid - each iterator that points to a rowid before iFrom. */ |
+ for(p=pTerm; p; p=p->pSynonym){ |
+ if( sqlite3Fts5IterEof(p->pIter)==0 ){ |
+ i64 ii = p->pIter->iRowid; |
+ if( ii==iRowid |
+ || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) |
+ ){ |
+ if( bFromValid ){ |
+ rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); |
+ }else{ |
+ rc = sqlite3Fts5IterNext(p->pIter); |
+ } |
+ if( rc!=SQLITE_OK ) break; |
+ if( sqlite3Fts5IterEof(p->pIter)==0 ){ |
+ bEof = 0; |
+ } |
+ }else{ |
+ bEof = 0; |
+ } |
+ } |
+ } |
+ |
+ /* Set the EOF flag if either all synonym iterators are at EOF or an |
+ ** error has occurred. */ |
+ pNode->bEof = (rc || bEof); |
+ }else{ |
+ Fts5IndexIter *pIter = pTerm->pIter; |
+ |
+ assert( Fts5NodeIsString(pNode) ); |
+ if( bFromValid ){ |
+ rc = sqlite3Fts5IterNextFrom(pIter, iFrom); |
+ }else{ |
+ rc = sqlite3Fts5IterNext(pIter); |
+ } |
+ |
+ pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); |
+ } |
+ |
+ if( pNode->bEof==0 ){ |
+ assert( rc==SQLITE_OK ); |
+ rc = fts5ExprNodeTest_STRING(pExpr, pNode); |
+ } |
+ |
+ return rc; |
+} |
+ |
+ |
+static int fts5ExprNodeTest_TERM( |
+ Fts5Expr *pExpr, /* Expression that pNear is a part of */ |
+ Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */ |
+){ |
+ /* As this "NEAR" object is actually a single phrase that consists |
+ ** of a single term only, grab pointers into the poslist managed by the |
+ ** fts5_index.c iterator object. This is much faster than synthesizing |
+ ** a new poslist the way we have to for more complicated phrase or NEAR |
+ ** expressions. */ |
+ Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0]; |
+ Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; |
+ |
+ assert( pNode->eType==FTS5_TERM ); |
+ assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 ); |
+ assert( pPhrase->aTerm[0].pSynonym==0 ); |
+ |
+ pPhrase->poslist.n = pIter->nData; |
+ if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){ |
+ pPhrase->poslist.p = (u8*)pIter->pData; |
+ } |
+ pNode->iRowid = pIter->iRowid; |
+ pNode->bNomatch = (pPhrase->poslist.n==0); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** xNext() method for a node of type FTS5_TERM. |
+*/ |
+static int fts5ExprNodeNext_TERM( |
+ Fts5Expr *pExpr, |
+ Fts5ExprNode *pNode, |
+ int bFromValid, |
+ i64 iFrom |
+){ |
+ int rc; |
+ Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; |
+ |
+ assert( pNode->bEof==0 ); |
+ if( bFromValid ){ |
+ rc = sqlite3Fts5IterNextFrom(pIter, iFrom); |
+ }else{ |
+ rc = sqlite3Fts5IterNext(pIter); |
+ } |
+ if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ |
+ rc = fts5ExprNodeTest_TERM(pExpr, pNode); |
+ }else{ |
+ pNode->bEof = 1; |
+ pNode->bNomatch = 0; |
+ } |
+ return rc; |
+} |
+ |
+static void fts5ExprNodeTest_OR( |
+ Fts5Expr *pExpr, /* Expression of which pNode is a part */ |
+ Fts5ExprNode *pNode /* Expression node to test */ |
+){ |
+ Fts5ExprNode *pNext = pNode->apChild[0]; |
+ int i; |
+ |
+ for(i=1; i<pNode->nChild; i++){ |
+ Fts5ExprNode *pChild = pNode->apChild[i]; |
+ int cmp = fts5NodeCompare(pExpr, pNext, pChild); |
+ if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){ |
+ pNext = pChild; |
+ } |
+ } |
+ pNode->iRowid = pNext->iRowid; |
+ pNode->bEof = pNext->bEof; |
+ pNode->bNomatch = pNext->bNomatch; |
+} |
+ |
+static int fts5ExprNodeNext_OR( |
+ Fts5Expr *pExpr, |
+ Fts5ExprNode *pNode, |
+ int bFromValid, |
+ i64 iFrom |
+){ |
+ int i; |
+ i64 iLast = pNode->iRowid; |
+ |
+ for(i=0; i<pNode->nChild; i++){ |
+ Fts5ExprNode *p1 = pNode->apChild[i]; |
+ assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 ); |
+ if( p1->bEof==0 ){ |
+ if( (p1->iRowid==iLast) |
+ || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) |
+ ){ |
+ int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); |
+ if( rc!=SQLITE_OK ) return rc; |
+ } |
+ } |
+ } |
+ |
+ fts5ExprNodeTest_OR(pExpr, pNode); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Argument pNode is an FTS5_AND node. |
+*/ |
+static int fts5ExprNodeTest_AND( |
+ Fts5Expr *pExpr, /* Expression pPhrase belongs to */ |
+ Fts5ExprNode *pAnd /* FTS5_AND node to advance */ |
+){ |
+ int iChild; |
+ i64 iLast = pAnd->iRowid; |
+ int rc = SQLITE_OK; |
+ int bMatch; |
+ |
+ assert( pAnd->bEof==0 ); |
+ do { |
+ pAnd->bNomatch = 0; |
+ bMatch = 1; |
+ for(iChild=0; iChild<pAnd->nChild; iChild++){ |
+ Fts5ExprNode *pChild = pAnd->apChild[iChild]; |
+ int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid); |
+ if( cmp>0 ){ |
+ /* Advance pChild until it points to iLast or laster */ |
+ rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); |
+ if( rc!=SQLITE_OK ) return rc; |
+ } |
+ |
+ /* If the child node is now at EOF, so is the parent AND node. Otherwise, |
+ ** the child node is guaranteed to have advanced at least as far as |
+ ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the |
+ ** new lastest rowid seen so far. */ |
+ assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 ); |
+ if( pChild->bEof ){ |
+ fts5ExprSetEof(pAnd); |
+ bMatch = 1; |
+ break; |
+ }else if( iLast!=pChild->iRowid ){ |
+ bMatch = 0; |
+ iLast = pChild->iRowid; |
+ } |
+ |
+ if( pChild->bNomatch ){ |
+ pAnd->bNomatch = 1; |
+ } |
+ } |
+ }while( bMatch==0 ); |
+ |
+ if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){ |
+ fts5ExprNodeZeroPoslist(pAnd); |
+ } |
+ pAnd->iRowid = iLast; |
+ return SQLITE_OK; |
+} |
+ |
+static int fts5ExprNodeNext_AND( |
+ Fts5Expr *pExpr, |
+ Fts5ExprNode *pNode, |
+ int bFromValid, |
+ i64 iFrom |
+){ |
+ int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5ExprNodeTest_AND(pExpr, pNode); |
+ } |
+ return rc; |
+} |
+ |
+static int fts5ExprNodeTest_NOT( |
+ Fts5Expr *pExpr, /* Expression pPhrase belongs to */ |
+ Fts5ExprNode *pNode /* FTS5_NOT node to advance */ |
+){ |
+ int rc = SQLITE_OK; |
+ Fts5ExprNode *p1 = pNode->apChild[0]; |
+ Fts5ExprNode *p2 = pNode->apChild[1]; |
+ assert( pNode->nChild==2 ); |
+ |
+ while( rc==SQLITE_OK && p1->bEof==0 ){ |
+ int cmp = fts5NodeCompare(pExpr, p1, p2); |
+ if( cmp>0 ){ |
+ rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid); |
+ cmp = fts5NodeCompare(pExpr, p1, p2); |
+ } |
+ assert( rc!=SQLITE_OK || cmp<=0 ); |
+ if( cmp || p2->bNomatch ) break; |
+ rc = fts5ExprNodeNext(pExpr, p1, 0, 0); |
+ } |
+ pNode->bEof = p1->bEof; |
+ pNode->bNomatch = p1->bNomatch; |
+ pNode->iRowid = p1->iRowid; |
+ if( p1->bEof ){ |
+ fts5ExprNodeZeroPoslist(p2); |
+ } |
+ return rc; |
+} |
+ |
+static int fts5ExprNodeNext_NOT( |
+ Fts5Expr *pExpr, |
+ Fts5ExprNode *pNode, |
+ int bFromValid, |
+ i64 iFrom |
+){ |
+ int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5ExprNodeTest_NOT(pExpr, pNode); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** If pNode currently points to a match, this function returns SQLITE_OK |
+** without modifying it. Otherwise, pNode is advanced until it does point |
+** to a match or EOF is reached. |
+*/ |
+static int fts5ExprNodeTest( |
+ Fts5Expr *pExpr, /* Expression of which pNode is a part */ |
+ Fts5ExprNode *pNode /* Expression node to test */ |
+){ |
+ int rc = SQLITE_OK; |
+ if( pNode->bEof==0 ){ |
+ switch( pNode->eType ){ |
+ |
+ case FTS5_STRING: { |
+ rc = fts5ExprNodeTest_STRING(pExpr, pNode); |
+ break; |
+ } |
+ |
+ case FTS5_TERM: { |
+ rc = fts5ExprNodeTest_TERM(pExpr, pNode); |
+ break; |
+ } |
+ |
+ case FTS5_AND: { |
+ rc = fts5ExprNodeTest_AND(pExpr, pNode); |
+ break; |
+ } |
+ |
+ case FTS5_OR: { |
+ fts5ExprNodeTest_OR(pExpr, pNode); |
+ break; |
+ } |
+ |
+ default: assert( pNode->eType==FTS5_NOT ); { |
+ rc = fts5ExprNodeTest_NOT(pExpr, pNode); |
+ break; |
+ } |
+ } |
+ } |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Set node pNode, which is part of expression pExpr, to point to the first |
+** match. If there are no matches, set the Node.bEof flag to indicate EOF. |
+** |
+** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. |
+** It is not an error if there are no matches. |
+*/ |
+static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ |
+ int rc = SQLITE_OK; |
+ pNode->bEof = 0; |
+ pNode->bNomatch = 0; |
+ |
+ if( Fts5NodeIsString(pNode) ){ |
+ /* Initialize all term iterators in the NEAR object. */ |
+ rc = fts5ExprNearInitAll(pExpr, pNode); |
+ }else if( pNode->xNext==0 ){ |
+ pNode->bEof = 1; |
+ }else{ |
+ int i; |
+ int nEof = 0; |
+ for(i=0; i<pNode->nChild && rc==SQLITE_OK; i++){ |
+ Fts5ExprNode *pChild = pNode->apChild[i]; |
+ rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]); |
+ assert( pChild->bEof==0 || pChild->bEof==1 ); |
+ nEof += pChild->bEof; |
+ } |
+ pNode->iRowid = pNode->apChild[0]->iRowid; |
+ |
+ switch( pNode->eType ){ |
+ case FTS5_AND: |
+ if( nEof>0 ) fts5ExprSetEof(pNode); |
+ break; |
+ |
+ case FTS5_OR: |
+ if( pNode->nChild==nEof ) fts5ExprSetEof(pNode); |
+ break; |
+ |
+ default: |
+ assert( pNode->eType==FTS5_NOT ); |
+ pNode->bEof = pNode->apChild[0]->bEof; |
+ break; |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5ExprNodeTest(pExpr, pNode); |
+ } |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Begin iterating through the set of documents in index pIdx matched by |
+** the MATCH expression passed as the first argument. If the "bDesc" |
+** parameter is passed a non-zero value, iteration is in descending rowid |
+** order. Or, if it is zero, in ascending order. |
+** |
+** If iterating in ascending rowid order (bDesc==0), the first document |
+** visited is that with the smallest rowid that is larger than or equal |
+** to parameter iFirst. Or, if iterating in ascending order (bDesc==1), |
+** then the first document visited must have a rowid smaller than or |
+** equal to iFirst. |
+** |
+** Return SQLITE_OK if successful, or an SQLite error code otherwise. It |
+** is not considered an error if the query does not match any documents. |
+*/ |
+static int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ |
+ Fts5ExprNode *pRoot = p->pRoot; |
+ int rc; /* Return code */ |
+ |
+ p->pIndex = pIdx; |
+ p->bDesc = bDesc; |
+ rc = fts5ExprNodeFirst(p, pRoot); |
+ |
+ /* If not at EOF but the current rowid occurs earlier than iFirst in |
+ ** the iteration order, move to document iFirst or later. */ |
+ if( rc==SQLITE_OK |
+ && 0==pRoot->bEof |
+ && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 |
+ ){ |
+ rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); |
+ } |
+ |
+ /* If the iterator is not at a real match, skip forward until it is. */ |
+ while( pRoot->bNomatch ){ |
+ assert( pRoot->bEof==0 && rc==SQLITE_OK ); |
+ rc = fts5ExprNodeNext(p, pRoot, 0, 0); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Move to the next document |
+** |
+** Return SQLITE_OK if successful, or an SQLite error code otherwise. It |
+** is not considered an error if the query does not match any documents. |
+*/ |
+static int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){ |
+ int rc; |
+ Fts5ExprNode *pRoot = p->pRoot; |
+ assert( pRoot->bEof==0 && pRoot->bNomatch==0 ); |
+ do { |
+ rc = fts5ExprNodeNext(p, pRoot, 0, 0); |
+ assert( pRoot->bNomatch==0 || (rc==SQLITE_OK && pRoot->bEof==0) ); |
+ }while( pRoot->bNomatch ); |
+ if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){ |
+ pRoot->bEof = 1; |
+ } |
+ return rc; |
+} |
+ |
+static int sqlite3Fts5ExprEof(Fts5Expr *p){ |
+ return p->pRoot->bEof; |
+} |
+ |
+static i64 sqlite3Fts5ExprRowid(Fts5Expr *p){ |
+ return p->pRoot->iRowid; |
+} |
+ |
+static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){ |
+ int rc = SQLITE_OK; |
+ *pz = sqlite3Fts5Strndup(&rc, pToken->p, pToken->n); |
+ return rc; |
+} |
+ |
+/* |
+** Free the phrase object passed as the only argument. |
+*/ |
+static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ |
+ if( pPhrase ){ |
+ int i; |
+ for(i=0; i<pPhrase->nTerm; i++){ |
+ Fts5ExprTerm *pSyn; |
+ Fts5ExprTerm *pNext; |
+ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; |
+ sqlite3_free(pTerm->zTerm); |
+ sqlite3Fts5IterClose(pTerm->pIter); |
+ for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ |
+ pNext = pSyn->pSynonym; |
+ sqlite3Fts5IterClose(pSyn->pIter); |
+ fts5BufferFree((Fts5Buffer*)&pSyn[1]); |
+ sqlite3_free(pSyn); |
+ } |
+ } |
+ if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist); |
+ sqlite3_free(pPhrase); |
+ } |
+} |
+ |
+/* |
+** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated |
+** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is |
+** appended to it and the results returned. |
+** |
+** If an OOM error occurs, both the pNear and pPhrase objects are freed and |
+** NULL returned. |
+*/ |
+static Fts5ExprNearset *sqlite3Fts5ParseNearset( |
+ Fts5Parse *pParse, /* Parse context */ |
+ Fts5ExprNearset *pNear, /* Existing nearset, or NULL */ |
+ Fts5ExprPhrase *pPhrase /* Recently parsed phrase */ |
+){ |
+ const int SZALLOC = 8; |
+ Fts5ExprNearset *pRet = 0; |
+ |
+ if( pParse->rc==SQLITE_OK ){ |
+ if( pPhrase==0 ){ |
+ return pNear; |
+ } |
+ if( pNear==0 ){ |
+ int nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); |
+ pRet = sqlite3_malloc(nByte); |
+ if( pRet==0 ){ |
+ pParse->rc = SQLITE_NOMEM; |
+ }else{ |
+ memset(pRet, 0, nByte); |
+ } |
+ }else if( (pNear->nPhrase % SZALLOC)==0 ){ |
+ int nNew = pNear->nPhrase + SZALLOC; |
+ int nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*); |
+ |
+ pRet = (Fts5ExprNearset*)sqlite3_realloc(pNear, nByte); |
+ if( pRet==0 ){ |
+ pParse->rc = SQLITE_NOMEM; |
+ } |
+ }else{ |
+ pRet = pNear; |
+ } |
+ } |
+ |
+ if( pRet==0 ){ |
+ assert( pParse->rc!=SQLITE_OK ); |
+ sqlite3Fts5ParseNearsetFree(pNear); |
+ sqlite3Fts5ParsePhraseFree(pPhrase); |
+ }else{ |
+ if( pRet->nPhrase>0 ){ |
+ Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1]; |
+ assert( pLast==pParse->apPhrase[pParse->nPhrase-2] ); |
+ if( pPhrase->nTerm==0 ){ |
+ fts5ExprPhraseFree(pPhrase); |
+ pRet->nPhrase--; |
+ pParse->nPhrase--; |
+ pPhrase = pLast; |
+ }else if( pLast->nTerm==0 ){ |
+ fts5ExprPhraseFree(pLast); |
+ pParse->apPhrase[pParse->nPhrase-2] = pPhrase; |
+ pParse->nPhrase--; |
+ pRet->nPhrase--; |
+ } |
+ } |
+ pRet->apPhrase[pRet->nPhrase++] = pPhrase; |
+ } |
+ return pRet; |
+} |
+ |
+typedef struct TokenCtx TokenCtx; |
+struct TokenCtx { |
+ Fts5ExprPhrase *pPhrase; |
+ int rc; |
+}; |
+ |
+/* |
+** Callback for tokenizing terms used by ParseTerm(). |
+*/ |
+static int fts5ParseTokenize( |
+ void *pContext, /* Pointer to Fts5InsertCtx object */ |
+ int tflags, /* Mask of FTS5_TOKEN_* flags */ |
+ const char *pToken, /* Buffer containing token */ |
+ int nToken, /* Size of token in bytes */ |
+ int iUnused1, /* Start offset of token */ |
+ int iUnused2 /* End offset of token */ |
+){ |
+ int rc = SQLITE_OK; |
+ const int SZALLOC = 8; |
+ TokenCtx *pCtx = (TokenCtx*)pContext; |
+ Fts5ExprPhrase *pPhrase = pCtx->pPhrase; |
+ |
+ UNUSED_PARAM2(iUnused1, iUnused2); |
+ |
+ /* If an error has already occurred, this is a no-op */ |
+ if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; |
+ if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; |
+ |
+ if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){ |
+ Fts5ExprTerm *pSyn; |
+ int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1; |
+ pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); |
+ if( pSyn==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ memset(pSyn, 0, nByte); |
+ pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); |
+ memcpy(pSyn->zTerm, pToken, nToken); |
+ pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; |
+ pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; |
+ } |
+ }else{ |
+ Fts5ExprTerm *pTerm; |
+ if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ |
+ Fts5ExprPhrase *pNew; |
+ int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); |
+ |
+ pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, |
+ sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew |
+ ); |
+ if( pNew==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); |
+ pCtx->pPhrase = pPhrase = pNew; |
+ pNew->nTerm = nNew - SZALLOC; |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; |
+ memset(pTerm, 0, sizeof(Fts5ExprTerm)); |
+ pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); |
+ } |
+ } |
+ |
+ pCtx->rc = rc; |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Free the phrase object passed as the only argument. |
+*/ |
+static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase *pPhrase){ |
+ fts5ExprPhraseFree(pPhrase); |
+} |
+ |
+/* |
+** Free the phrase object passed as the second argument. |
+*/ |
+static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset *pNear){ |
+ if( pNear ){ |
+ int i; |
+ for(i=0; i<pNear->nPhrase; i++){ |
+ fts5ExprPhraseFree(pNear->apPhrase[i]); |
+ } |
+ sqlite3_free(pNear->pColset); |
+ sqlite3_free(pNear); |
+ } |
+} |
+ |
+static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p){ |
+ assert( pParse->pExpr==0 ); |
+ pParse->pExpr = p; |
+} |
+ |
+/* |
+** This function is called by the parser to process a string token. The |
+** string may or may not be quoted. In any case it is tokenized and a |
+** phrase object consisting of all tokens returned. |
+*/ |
+static Fts5ExprPhrase *sqlite3Fts5ParseTerm( |
+ Fts5Parse *pParse, /* Parse context */ |
+ Fts5ExprPhrase *pAppend, /* Phrase to append to */ |
+ Fts5Token *pToken, /* String to tokenize */ |
+ int bPrefix /* True if there is a trailing "*" */ |
+){ |
+ Fts5Config *pConfig = pParse->pConfig; |
+ TokenCtx sCtx; /* Context object passed to callback */ |
+ int rc; /* Tokenize return code */ |
+ char *z = 0; |
+ |
+ memset(&sCtx, 0, sizeof(TokenCtx)); |
+ sCtx.pPhrase = pAppend; |
+ |
+ rc = fts5ParseStringFromToken(pToken, &z); |
+ if( rc==SQLITE_OK ){ |
+ int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0); |
+ int n; |
+ sqlite3Fts5Dequote(z); |
+ n = (int)strlen(z); |
+ rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); |
+ } |
+ sqlite3_free(z); |
+ if( rc || (rc = sCtx.rc) ){ |
+ pParse->rc = rc; |
+ fts5ExprPhraseFree(sCtx.pPhrase); |
+ sCtx.pPhrase = 0; |
+ }else{ |
+ |
+ if( pAppend==0 ){ |
+ if( (pParse->nPhrase % 8)==0 ){ |
+ int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8); |
+ Fts5ExprPhrase **apNew; |
+ apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte); |
+ if( apNew==0 ){ |
+ pParse->rc = SQLITE_NOMEM; |
+ fts5ExprPhraseFree(sCtx.pPhrase); |
+ return 0; |
+ } |
+ pParse->apPhrase = apNew; |
+ } |
+ pParse->nPhrase++; |
+ } |
+ |
+ if( sCtx.pPhrase==0 ){ |
+ /* This happens when parsing a token or quoted phrase that contains |
+ ** no token characters at all. (e.g ... MATCH '""'). */ |
+ sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase)); |
+ }else if( sCtx.pPhrase->nTerm ){ |
+ sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix; |
+ } |
+ pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase; |
+ } |
+ |
+ return sCtx.pPhrase; |
+} |
+ |
+/* |
+** Create a new FTS5 expression by cloning phrase iPhrase of the |
+** expression passed as the second argument. |
+*/ |
+static int sqlite3Fts5ExprClonePhrase( |
+ Fts5Expr *pExpr, |
+ int iPhrase, |
+ Fts5Expr **ppNew |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ |
+ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ |
+ TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ |
+ |
+ pOrig = pExpr->apExprPhrase[iPhrase]; |
+ pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); |
+ if( rc==SQLITE_OK ){ |
+ pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, |
+ sizeof(Fts5ExprPhrase*)); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, |
+ sizeof(Fts5ExprNode)); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, |
+ sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; |
+ if( pColsetOrig ){ |
+ int nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); |
+ Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); |
+ if( pColset ){ |
+ memcpy(pColset, pColsetOrig, nByte); |
+ } |
+ pNew->pRoot->pNear->pColset = pColset; |
+ } |
+ } |
+ |
+ if( pOrig->nTerm ){ |
+ int i; /* Used to iterate through phrase terms */ |
+ for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){ |
+ int tflags = 0; |
+ Fts5ExprTerm *p; |
+ for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ |
+ const char *zTerm = p->zTerm; |
+ rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), |
+ 0, 0); |
+ tflags = FTS5_TOKEN_COLOCATED; |
+ } |
+ if( rc==SQLITE_OK ){ |
+ sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; |
+ } |
+ } |
+ }else{ |
+ /* This happens when parsing a token or quoted phrase that contains |
+ ** no token characters at all. (e.g ... MATCH '""'). */ |
+ sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ /* All the allocations succeeded. Put the expression object together. */ |
+ pNew->pIndex = pExpr->pIndex; |
+ pNew->pConfig = pExpr->pConfig; |
+ pNew->nPhrase = 1; |
+ pNew->apExprPhrase[0] = sCtx.pPhrase; |
+ pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; |
+ pNew->pRoot->pNear->nPhrase = 1; |
+ sCtx.pPhrase->pNode = pNew->pRoot; |
+ |
+ if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ |
+ pNew->pRoot->eType = FTS5_TERM; |
+ pNew->pRoot->xNext = fts5ExprNodeNext_TERM; |
+ }else{ |
+ pNew->pRoot->eType = FTS5_STRING; |
+ pNew->pRoot->xNext = fts5ExprNodeNext_STRING; |
+ } |
+ }else{ |
+ sqlite3Fts5ExprFree(pNew); |
+ fts5ExprPhraseFree(sCtx.pPhrase); |
+ pNew = 0; |
+ } |
+ |
+ *ppNew = pNew; |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Token pTok has appeared in a MATCH expression where the NEAR operator |
+** is expected. If token pTok does not contain "NEAR", store an error |
+** in the pParse object. |
+*/ |
+static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token *pTok){ |
+ if( pTok->n!=4 || memcmp("NEAR", pTok->p, 4) ){ |
+ sqlite3Fts5ParseError( |
+ pParse, "fts5: syntax error near \"%.*s\"", pTok->n, pTok->p |
+ ); |
+ } |
+} |
+ |
+static void sqlite3Fts5ParseSetDistance( |
+ Fts5Parse *pParse, |
+ Fts5ExprNearset *pNear, |
+ Fts5Token *p |
+){ |
+ if( pNear ){ |
+ int nNear = 0; |
+ int i; |
+ if( p->n ){ |
+ for(i=0; i<p->n; i++){ |
+ char c = (char)p->p[i]; |
+ if( c<'0' || c>'9' ){ |
+ sqlite3Fts5ParseError( |
+ pParse, "expected integer, got \"%.*s\"", p->n, p->p |
+ ); |
+ return; |
+ } |
+ nNear = nNear * 10 + (p->p[i] - '0'); |
+ } |
+ }else{ |
+ nNear = FTS5_DEFAULT_NEARDIST; |
+ } |
+ pNear->nNear = nNear; |
+ } |
+} |
+ |
+/* |
+** The second argument passed to this function may be NULL, or it may be |
+** an existing Fts5Colset object. This function returns a pointer to |
+** a new colset object containing the contents of (p) with new value column |
+** number iCol appended. |
+** |
+** If an OOM error occurs, store an error code in pParse and return NULL. |
+** The old colset object (if any) is not freed in this case. |
+*/ |
+static Fts5Colset *fts5ParseColset( |
+ Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ |
+ Fts5Colset *p, /* Existing colset object */ |
+ int iCol /* New column to add to colset object */ |
+){ |
+ int nCol = p ? p->nCol : 0; /* Num. columns already in colset object */ |
+ Fts5Colset *pNew; /* New colset object to return */ |
+ |
+ assert( pParse->rc==SQLITE_OK ); |
+ assert( iCol>=0 && iCol<pParse->pConfig->nCol ); |
+ |
+ pNew = sqlite3_realloc(p, sizeof(Fts5Colset) + sizeof(int)*nCol); |
+ if( pNew==0 ){ |
+ pParse->rc = SQLITE_NOMEM; |
+ }else{ |
+ int *aiCol = pNew->aiCol; |
+ int i, j; |
+ for(i=0; i<nCol; i++){ |
+ if( aiCol[i]==iCol ) return pNew; |
+ if( aiCol[i]>iCol ) break; |
+ } |
+ for(j=nCol; j>i; j--){ |
+ aiCol[j] = aiCol[j-1]; |
+ } |
+ aiCol[i] = iCol; |
+ pNew->nCol = nCol+1; |
+ |
+#ifndef NDEBUG |
+ /* Check that the array is in order and contains no duplicate entries. */ |
+ for(i=1; i<pNew->nCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] ); |
+#endif |
+ } |
+ |
+ return pNew; |
+} |
+ |
+/* |
+** Allocate and return an Fts5Colset object specifying the inverse of |
+** the colset passed as the second argument. Free the colset passed |
+** as the second argument before returning. |
+*/ |
+static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){ |
+ Fts5Colset *pRet; |
+ int nCol = pParse->pConfig->nCol; |
+ |
+ pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, |
+ sizeof(Fts5Colset) + sizeof(int)*nCol |
+ ); |
+ if( pRet ){ |
+ int i; |
+ int iOld = 0; |
+ for(i=0; i<nCol; i++){ |
+ if( iOld>=p->nCol || p->aiCol[iOld]!=i ){ |
+ pRet->aiCol[pRet->nCol++] = i; |
+ }else{ |
+ iOld++; |
+ } |
+ } |
+ } |
+ |
+ sqlite3_free(p); |
+ return pRet; |
+} |
+ |
+static Fts5Colset *sqlite3Fts5ParseColset( |
+ Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ |
+ Fts5Colset *pColset, /* Existing colset object */ |
+ Fts5Token *p |
+){ |
+ Fts5Colset *pRet = 0; |
+ int iCol; |
+ char *z; /* Dequoted copy of token p */ |
+ |
+ z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n); |
+ if( pParse->rc==SQLITE_OK ){ |
+ Fts5Config *pConfig = pParse->pConfig; |
+ sqlite3Fts5Dequote(z); |
+ for(iCol=0; iCol<pConfig->nCol; iCol++){ |
+ if( 0==sqlite3_stricmp(pConfig->azCol[iCol], z) ) break; |
+ } |
+ if( iCol==pConfig->nCol ){ |
+ sqlite3Fts5ParseError(pParse, "no such column: %s", z); |
+ }else{ |
+ pRet = fts5ParseColset(pParse, pColset, iCol); |
+ } |
+ sqlite3_free(z); |
+ } |
+ |
+ if( pRet==0 ){ |
+ assert( pParse->rc!=SQLITE_OK ); |
+ sqlite3_free(pColset); |
+ } |
+ |
+ return pRet; |
+} |
+ |
+static void sqlite3Fts5ParseSetColset( |
+ Fts5Parse *pParse, |
+ Fts5ExprNearset *pNear, |
+ Fts5Colset *pColset |
+){ |
+ if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ |
+ pParse->rc = SQLITE_ERROR; |
+ pParse->zErr = sqlite3_mprintf( |
+ "fts5: column queries are not supported (detail=none)" |
+ ); |
+ sqlite3_free(pColset); |
+ return; |
+ } |
+ |
+ if( pNear ){ |
+ pNear->pColset = pColset; |
+ }else{ |
+ sqlite3_free(pColset); |
+ } |
+} |
+ |
+static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ |
+ switch( pNode->eType ){ |
+ case FTS5_STRING: { |
+ Fts5ExprNearset *pNear = pNode->pNear; |
+ if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 |
+ && pNear->apPhrase[0]->aTerm[0].pSynonym==0 |
+ ){ |
+ pNode->eType = FTS5_TERM; |
+ pNode->xNext = fts5ExprNodeNext_TERM; |
+ }else{ |
+ pNode->xNext = fts5ExprNodeNext_STRING; |
+ } |
+ break; |
+ }; |
+ |
+ case FTS5_OR: { |
+ pNode->xNext = fts5ExprNodeNext_OR; |
+ break; |
+ }; |
+ |
+ case FTS5_AND: { |
+ pNode->xNext = fts5ExprNodeNext_AND; |
+ break; |
+ }; |
+ |
+ default: assert( pNode->eType==FTS5_NOT ); { |
+ pNode->xNext = fts5ExprNodeNext_NOT; |
+ break; |
+ }; |
+ } |
+} |
+ |
+static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ |
+ if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ |
+ int nByte = sizeof(Fts5ExprNode*) * pSub->nChild; |
+ memcpy(&p->apChild[p->nChild], pSub->apChild, nByte); |
+ p->nChild += pSub->nChild; |
+ sqlite3_free(pSub); |
+ }else{ |
+ p->apChild[p->nChild++] = pSub; |
+ } |
+} |
+ |
+/* |
+** Allocate and return a new expression object. If anything goes wrong (i.e. |
+** OOM error), leave an error code in pParse and return NULL. |
+*/ |
+static Fts5ExprNode *sqlite3Fts5ParseNode( |
+ Fts5Parse *pParse, /* Parse context */ |
+ int eType, /* FTS5_STRING, AND, OR or NOT */ |
+ Fts5ExprNode *pLeft, /* Left hand child expression */ |
+ Fts5ExprNode *pRight, /* Right hand child expression */ |
+ Fts5ExprNearset *pNear /* For STRING expressions, the near cluster */ |
+){ |
+ Fts5ExprNode *pRet = 0; |
+ |
+ if( pParse->rc==SQLITE_OK ){ |
+ int nChild = 0; /* Number of children of returned node */ |
+ int nByte; /* Bytes of space to allocate for this node */ |
+ |
+ assert( (eType!=FTS5_STRING && !pNear) |
+ || (eType==FTS5_STRING && !pLeft && !pRight) |
+ ); |
+ if( eType==FTS5_STRING && pNear==0 ) return 0; |
+ if( eType!=FTS5_STRING && pLeft==0 ) return pRight; |
+ if( eType!=FTS5_STRING && pRight==0 ) return pLeft; |
+ |
+ if( eType==FTS5_NOT ){ |
+ nChild = 2; |
+ }else if( eType==FTS5_AND || eType==FTS5_OR ){ |
+ nChild = 2; |
+ if( pLeft->eType==eType ) nChild += pLeft->nChild-1; |
+ if( pRight->eType==eType ) nChild += pRight->nChild-1; |
+ } |
+ |
+ nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); |
+ pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); |
+ |
+ if( pRet ){ |
+ pRet->eType = eType; |
+ pRet->pNear = pNear; |
+ fts5ExprAssignXNext(pRet); |
+ if( eType==FTS5_STRING ){ |
+ int iPhrase; |
+ for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){ |
+ pNear->apPhrase[iPhrase]->pNode = pRet; |
+ if( pNear->apPhrase[iPhrase]->nTerm==0 ){ |
+ pRet->xNext = 0; |
+ pRet->eType = FTS5_EOF; |
+ } |
+ } |
+ |
+ if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL |
+ && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1) |
+ ){ |
+ assert( pParse->rc==SQLITE_OK ); |
+ pParse->rc = SQLITE_ERROR; |
+ assert( pParse->zErr==0 ); |
+ pParse->zErr = sqlite3_mprintf( |
+ "fts5: %s queries are not supported (detail!=full)", |
+ pNear->nPhrase==1 ? "phrase": "NEAR" |
+ ); |
+ sqlite3_free(pRet); |
+ pRet = 0; |
+ } |
+ |
+ }else{ |
+ fts5ExprAddChildren(pRet, pLeft); |
+ fts5ExprAddChildren(pRet, pRight); |
+ } |
+ } |
+ } |
+ |
+ if( pRet==0 ){ |
+ assert( pParse->rc!=SQLITE_OK ); |
+ sqlite3Fts5ParseNodeFree(pLeft); |
+ sqlite3Fts5ParseNodeFree(pRight); |
+ sqlite3Fts5ParseNearsetFree(pNear); |
+ } |
+ return pRet; |
+} |
+ |
+static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( |
+ Fts5Parse *pParse, /* Parse context */ |
+ Fts5ExprNode *pLeft, /* Left hand child expression */ |
+ Fts5ExprNode *pRight /* Right hand child expression */ |
+){ |
+ Fts5ExprNode *pRet = 0; |
+ Fts5ExprNode *pPrev; |
+ |
+ if( pParse->rc ){ |
+ sqlite3Fts5ParseNodeFree(pLeft); |
+ sqlite3Fts5ParseNodeFree(pRight); |
+ }else{ |
+ |
+ assert( pLeft->eType==FTS5_STRING |
+ || pLeft->eType==FTS5_TERM |
+ || pLeft->eType==FTS5_EOF |
+ || pLeft->eType==FTS5_AND |
+ ); |
+ assert( pRight->eType==FTS5_STRING |
+ || pRight->eType==FTS5_TERM |
+ || pRight->eType==FTS5_EOF |
+ ); |
+ |
+ if( pLeft->eType==FTS5_AND ){ |
+ pPrev = pLeft->apChild[pLeft->nChild-1]; |
+ }else{ |
+ pPrev = pLeft; |
+ } |
+ assert( pPrev->eType==FTS5_STRING |
+ || pPrev->eType==FTS5_TERM |
+ || pPrev->eType==FTS5_EOF |
+ ); |
+ |
+ if( pRight->eType==FTS5_EOF ){ |
+ assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] ); |
+ sqlite3Fts5ParseNodeFree(pRight); |
+ pRet = pLeft; |
+ pParse->nPhrase--; |
+ } |
+ else if( pPrev->eType==FTS5_EOF ){ |
+ Fts5ExprPhrase **ap; |
+ |
+ if( pPrev==pLeft ){ |
+ pRet = pRight; |
+ }else{ |
+ pLeft->apChild[pLeft->nChild-1] = pRight; |
+ pRet = pLeft; |
+ } |
+ |
+ ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase]; |
+ assert( ap[0]==pPrev->pNear->apPhrase[0] ); |
+ memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase); |
+ pParse->nPhrase--; |
+ |
+ sqlite3Fts5ParseNodeFree(pPrev); |
+ } |
+ else{ |
+ pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND, pLeft, pRight, 0); |
+ } |
+ } |
+ |
+ return pRet; |
+} |
+ |
+static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ |
+ int nByte = 0; |
+ Fts5ExprTerm *p; |
+ char *zQuoted; |
+ |
+ /* Determine the maximum amount of space required. */ |
+ for(p=pTerm; p; p=p->pSynonym){ |
+ nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2; |
+ } |
+ zQuoted = sqlite3_malloc(nByte); |
+ |
+ if( zQuoted ){ |
+ int i = 0; |
+ for(p=pTerm; p; p=p->pSynonym){ |
+ char *zIn = p->zTerm; |
+ zQuoted[i++] = '"'; |
+ while( *zIn ){ |
+ if( *zIn=='"' ) zQuoted[i++] = '"'; |
+ zQuoted[i++] = *zIn++; |
+ } |
+ zQuoted[i++] = '"'; |
+ if( p->pSynonym ) zQuoted[i++] = '|'; |
+ } |
+ if( pTerm->bPrefix ){ |
+ zQuoted[i++] = ' '; |
+ zQuoted[i++] = '*'; |
+ } |
+ zQuoted[i++] = '\0'; |
+ } |
+ return zQuoted; |
+} |
+ |
+static char *fts5PrintfAppend(char *zApp, const char *zFmt, ...){ |
+ char *zNew; |
+ va_list ap; |
+ va_start(ap, zFmt); |
+ zNew = sqlite3_vmprintf(zFmt, ap); |
+ va_end(ap); |
+ if( zApp && zNew ){ |
+ char *zNew2 = sqlite3_mprintf("%s%s", zApp, zNew); |
+ sqlite3_free(zNew); |
+ zNew = zNew2; |
+ } |
+ sqlite3_free(zApp); |
+ return zNew; |
+} |
+ |
+/* |
+** Compose a tcl-readable representation of expression pExpr. Return a |
+** pointer to a buffer containing that representation. It is the |
+** responsibility of the caller to at some point free the buffer using |
+** sqlite3_free(). |
+*/ |
+static char *fts5ExprPrintTcl( |
+ Fts5Config *pConfig, |
+ const char *zNearsetCmd, |
+ Fts5ExprNode *pExpr |
+){ |
+ char *zRet = 0; |
+ if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ |
+ Fts5ExprNearset *pNear = pExpr->pNear; |
+ int i; |
+ int iTerm; |
+ |
+ zRet = fts5PrintfAppend(zRet, "%s ", zNearsetCmd); |
+ if( zRet==0 ) return 0; |
+ if( pNear->pColset ){ |
+ int *aiCol = pNear->pColset->aiCol; |
+ int nCol = pNear->pColset->nCol; |
+ if( nCol==1 ){ |
+ zRet = fts5PrintfAppend(zRet, "-col %d ", aiCol[0]); |
+ }else{ |
+ zRet = fts5PrintfAppend(zRet, "-col {%d", aiCol[0]); |
+ for(i=1; i<pNear->pColset->nCol; i++){ |
+ zRet = fts5PrintfAppend(zRet, " %d", aiCol[i]); |
+ } |
+ zRet = fts5PrintfAppend(zRet, "} "); |
+ } |
+ if( zRet==0 ) return 0; |
+ } |
+ |
+ if( pNear->nPhrase>1 ){ |
+ zRet = fts5PrintfAppend(zRet, "-near %d ", pNear->nNear); |
+ if( zRet==0 ) return 0; |
+ } |
+ |
+ zRet = fts5PrintfAppend(zRet, "--"); |
+ if( zRet==0 ) return 0; |
+ |
+ for(i=0; i<pNear->nPhrase; i++){ |
+ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; |
+ |
+ zRet = fts5PrintfAppend(zRet, " {"); |
+ for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){ |
+ char *zTerm = pPhrase->aTerm[iTerm].zTerm; |
+ zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); |
+ if( pPhrase->aTerm[iTerm].bPrefix ){ |
+ zRet = fts5PrintfAppend(zRet, "*"); |
+ } |
+ } |
+ |
+ if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); |
+ if( zRet==0 ) return 0; |
+ } |
+ |
+ }else{ |
+ char const *zOp = 0; |
+ int i; |
+ switch( pExpr->eType ){ |
+ case FTS5_AND: zOp = "AND"; break; |
+ case FTS5_NOT: zOp = "NOT"; break; |
+ default: |
+ assert( pExpr->eType==FTS5_OR ); |
+ zOp = "OR"; |
+ break; |
+ } |
+ |
+ zRet = sqlite3_mprintf("%s", zOp); |
+ for(i=0; zRet && i<pExpr->nChild; i++){ |
+ char *z = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->apChild[i]); |
+ if( !z ){ |
+ sqlite3_free(zRet); |
+ zRet = 0; |
+ }else{ |
+ zRet = fts5PrintfAppend(zRet, " [%z]", z); |
+ } |
+ } |
+ } |
+ |
+ return zRet; |
+} |
+ |
+static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){ |
+ char *zRet = 0; |
+ if( pExpr->eType==0 ){ |
+ return sqlite3_mprintf("\"\""); |
+ }else |
+ if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ |
+ Fts5ExprNearset *pNear = pExpr->pNear; |
+ int i; |
+ int iTerm; |
+ |
+ if( pNear->pColset ){ |
+ int iCol = pNear->pColset->aiCol[0]; |
+ zRet = fts5PrintfAppend(zRet, "%s : ", pConfig->azCol[iCol]); |
+ if( zRet==0 ) return 0; |
+ } |
+ |
+ if( pNear->nPhrase>1 ){ |
+ zRet = fts5PrintfAppend(zRet, "NEAR("); |
+ if( zRet==0 ) return 0; |
+ } |
+ |
+ for(i=0; i<pNear->nPhrase; i++){ |
+ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; |
+ if( i!=0 ){ |
+ zRet = fts5PrintfAppend(zRet, " "); |
+ if( zRet==0 ) return 0; |
+ } |
+ for(iTerm=0; iTerm<pPhrase->nTerm; iTerm++){ |
+ char *zTerm = fts5ExprTermPrint(&pPhrase->aTerm[iTerm]); |
+ if( zTerm ){ |
+ zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" + ", zTerm); |
+ sqlite3_free(zTerm); |
+ } |
+ if( zTerm==0 || zRet==0 ){ |
+ sqlite3_free(zRet); |
+ return 0; |
+ } |
+ } |
+ } |
+ |
+ if( pNear->nPhrase>1 ){ |
+ zRet = fts5PrintfAppend(zRet, ", %d)", pNear->nNear); |
+ if( zRet==0 ) return 0; |
+ } |
+ |
+ }else{ |
+ char const *zOp = 0; |
+ int i; |
+ |
+ switch( pExpr->eType ){ |
+ case FTS5_AND: zOp = " AND "; break; |
+ case FTS5_NOT: zOp = " NOT "; break; |
+ default: |
+ assert( pExpr->eType==FTS5_OR ); |
+ zOp = " OR "; |
+ break; |
+ } |
+ |
+ for(i=0; i<pExpr->nChild; i++){ |
+ char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]); |
+ if( z==0 ){ |
+ sqlite3_free(zRet); |
+ zRet = 0; |
+ }else{ |
+ int e = pExpr->apChild[i]->eType; |
+ int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF); |
+ zRet = fts5PrintfAppend(zRet, "%s%s%z%s", |
+ (i==0 ? "" : zOp), |
+ (b?"(":""), z, (b?")":"") |
+ ); |
+ } |
+ if( zRet==0 ) break; |
+ } |
+ } |
+ |
+ return zRet; |
+} |
+ |
+/* |
+** The implementation of user-defined scalar functions fts5_expr() (bTcl==0) |
+** and fts5_expr_tcl() (bTcl!=0). |
+*/ |
+static void fts5ExprFunction( |
+ sqlite3_context *pCtx, /* Function call context */ |
+ int nArg, /* Number of args */ |
+ sqlite3_value **apVal, /* Function arguments */ |
+ int bTcl |
+){ |
+ Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); |
+ sqlite3 *db = sqlite3_context_db_handle(pCtx); |
+ const char *zExpr = 0; |
+ char *zErr = 0; |
+ Fts5Expr *pExpr = 0; |
+ int rc; |
+ int i; |
+ |
+ const char **azConfig; /* Array of arguments for Fts5Config */ |
+ const char *zNearsetCmd = "nearset"; |
+ int nConfig; /* Size of azConfig[] */ |
+ Fts5Config *pConfig = 0; |
+ int iArg = 1; |
+ |
+ if( nArg<1 ){ |
+ zErr = sqlite3_mprintf("wrong number of arguments to function %s", |
+ bTcl ? "fts5_expr_tcl" : "fts5_expr" |
+ ); |
+ sqlite3_result_error(pCtx, zErr, -1); |
+ sqlite3_free(zErr); |
+ return; |
+ } |
+ |
+ if( bTcl && nArg>1 ){ |
+ zNearsetCmd = (const char*)sqlite3_value_text(apVal[1]); |
+ iArg = 2; |
+ } |
+ |
+ nConfig = 3 + (nArg-iArg); |
+ azConfig = (const char**)sqlite3_malloc(sizeof(char*) * nConfig); |
+ if( azConfig==0 ){ |
+ sqlite3_result_error_nomem(pCtx); |
+ return; |
+ } |
+ azConfig[0] = 0; |
+ azConfig[1] = "main"; |
+ azConfig[2] = "tbl"; |
+ for(i=3; iArg<nArg; iArg++){ |
+ azConfig[i++] = (const char*)sqlite3_value_text(apVal[iArg]); |
+ } |
+ |
+ zExpr = (const char*)sqlite3_value_text(apVal[0]); |
+ |
+ rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ char *zText; |
+ if( pExpr->pRoot->xNext==0 ){ |
+ zText = sqlite3_mprintf(""); |
+ }else if( bTcl ){ |
+ zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot); |
+ }else{ |
+ zText = fts5ExprPrint(pConfig, pExpr->pRoot); |
+ } |
+ if( zText==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT); |
+ sqlite3_free(zText); |
+ } |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ if( zErr ){ |
+ sqlite3_result_error(pCtx, zErr, -1); |
+ sqlite3_free(zErr); |
+ }else{ |
+ sqlite3_result_error_code(pCtx, rc); |
+ } |
+ } |
+ sqlite3_free((void *)azConfig); |
+ sqlite3Fts5ConfigFree(pConfig); |
+ sqlite3Fts5ExprFree(pExpr); |
+} |
+ |
+static void fts5ExprFunctionHr( |
+ sqlite3_context *pCtx, /* Function call context */ |
+ int nArg, /* Number of args */ |
+ sqlite3_value **apVal /* Function arguments */ |
+){ |
+ fts5ExprFunction(pCtx, nArg, apVal, 0); |
+} |
+static void fts5ExprFunctionTcl( |
+ sqlite3_context *pCtx, /* Function call context */ |
+ int nArg, /* Number of args */ |
+ sqlite3_value **apVal /* Function arguments */ |
+){ |
+ fts5ExprFunction(pCtx, nArg, apVal, 1); |
+} |
+ |
+/* |
+** The implementation of an SQLite user-defined-function that accepts a |
+** single integer as an argument. If the integer is an alpha-numeric |
+** unicode code point, 1 is returned. Otherwise 0. |
+*/ |
+static void fts5ExprIsAlnum( |
+ sqlite3_context *pCtx, /* Function call context */ |
+ int nArg, /* Number of args */ |
+ sqlite3_value **apVal /* Function arguments */ |
+){ |
+ int iCode; |
+ if( nArg!=1 ){ |
+ sqlite3_result_error(pCtx, |
+ "wrong number of arguments to function fts5_isalnum", -1 |
+ ); |
+ return; |
+ } |
+ iCode = sqlite3_value_int(apVal[0]); |
+ sqlite3_result_int(pCtx, sqlite3Fts5UnicodeIsalnum(iCode)); |
+} |
+ |
+static void fts5ExprFold( |
+ sqlite3_context *pCtx, /* Function call context */ |
+ int nArg, /* Number of args */ |
+ sqlite3_value **apVal /* Function arguments */ |
+){ |
+ if( nArg!=1 && nArg!=2 ){ |
+ sqlite3_result_error(pCtx, |
+ "wrong number of arguments to function fts5_fold", -1 |
+ ); |
+ }else{ |
+ int iCode; |
+ int bRemoveDiacritics = 0; |
+ iCode = sqlite3_value_int(apVal[0]); |
+ if( nArg==2 ) bRemoveDiacritics = sqlite3_value_int(apVal[1]); |
+ sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics)); |
+ } |
+} |
+ |
+/* |
+** This is called during initialization to register the fts5_expr() scalar |
+** UDF with the SQLite handle passed as the only argument. |
+*/ |
+static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ |
+ struct Fts5ExprFunc { |
+ const char *z; |
+ void (*x)(sqlite3_context*,int,sqlite3_value**); |
+ } aFunc[] = { |
+ { "fts5_expr", fts5ExprFunctionHr }, |
+ { "fts5_expr_tcl", fts5ExprFunctionTcl }, |
+ { "fts5_isalnum", fts5ExprIsAlnum }, |
+ { "fts5_fold", fts5ExprFold }, |
+ }; |
+ int i; |
+ int rc = SQLITE_OK; |
+ void *pCtx = (void*)pGlobal; |
+ |
+ for(i=0; rc==SQLITE_OK && i<ArraySize(aFunc); i++){ |
+ struct Fts5ExprFunc *p = &aFunc[i]; |
+ rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0); |
+ } |
+ |
+ /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */ |
+#ifndef NDEBUG |
+ (void)sqlite3Fts5ParserTrace; |
+#endif |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Return the number of phrases in expression pExpr. |
+*/ |
+static int sqlite3Fts5ExprPhraseCount(Fts5Expr *pExpr){ |
+ return (pExpr ? pExpr->nPhrase : 0); |
+} |
+ |
+/* |
+** Return the number of terms in the iPhrase'th phrase in pExpr. |
+*/ |
+static int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){ |
+ if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0; |
+ return pExpr->apExprPhrase[iPhrase]->nTerm; |
+} |
+ |
+/* |
+** This function is used to access the current position list for phrase |
+** iPhrase. |
+*/ |
+static int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){ |
+ int nRet; |
+ Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; |
+ Fts5ExprNode *pNode = pPhrase->pNode; |
+ if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){ |
+ *pa = pPhrase->poslist.p; |
+ nRet = pPhrase->poslist.n; |
+ }else{ |
+ *pa = 0; |
+ nRet = 0; |
+ } |
+ return nRet; |
+} |
+ |
+struct Fts5PoslistPopulator { |
+ Fts5PoslistWriter writer; |
+ int bOk; /* True if ok to populate */ |
+ int bMiss; |
+}; |
+ |
+static Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){ |
+ Fts5PoslistPopulator *pRet; |
+ pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); |
+ if( pRet ){ |
+ int i; |
+ memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); |
+ for(i=0; i<pExpr->nPhrase; i++){ |
+ Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; |
+ Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; |
+ assert( pExpr->apExprPhrase[i]->nTerm==1 ); |
+ if( bLive && |
+ (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof) |
+ ){ |
+ pRet[i].bMiss = 1; |
+ }else{ |
+ pBuf->n = 0; |
+ } |
+ } |
+ } |
+ return pRet; |
+} |
+ |
+struct Fts5ExprCtx { |
+ Fts5Expr *pExpr; |
+ Fts5PoslistPopulator *aPopulator; |
+ i64 iOff; |
+}; |
+typedef struct Fts5ExprCtx Fts5ExprCtx; |
+ |
+/* |
+** TODO: Make this more efficient! |
+*/ |
+static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ |
+ int i; |
+ for(i=0; i<pColset->nCol; i++){ |
+ if( pColset->aiCol[i]==iCol ) return 1; |
+ } |
+ return 0; |
+} |
+ |
+static int fts5ExprPopulatePoslistsCb( |
+ void *pCtx, /* Copy of 2nd argument to xTokenize() */ |
+ int tflags, /* Mask of FTS5_TOKEN_* flags */ |
+ const char *pToken, /* Pointer to buffer containing token */ |
+ int nToken, /* Size of token in bytes */ |
+ int iUnused1, /* Byte offset of token within input text */ |
+ int iUnused2 /* Byte offset of end of token within input text */ |
+){ |
+ Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx; |
+ Fts5Expr *pExpr = p->pExpr; |
+ int i; |
+ |
+ UNUSED_PARAM2(iUnused1, iUnused2); |
+ |
+ if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; |
+ if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; |
+ for(i=0; i<pExpr->nPhrase; i++){ |
+ Fts5ExprTerm *pTerm; |
+ if( p->aPopulator[i].bOk==0 ) continue; |
+ for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ |
+ int nTerm = (int)strlen(pTerm->zTerm); |
+ if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix)) |
+ && memcmp(pTerm->zTerm, pToken, nTerm)==0 |
+ ){ |
+ int rc = sqlite3Fts5PoslistWriterAppend( |
+ &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff |
+ ); |
+ if( rc ) return rc; |
+ break; |
+ } |
+ } |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+static int sqlite3Fts5ExprPopulatePoslists( |
+ Fts5Config *pConfig, |
+ Fts5Expr *pExpr, |
+ Fts5PoslistPopulator *aPopulator, |
+ int iCol, |
+ const char *z, int n |
+){ |
+ int i; |
+ Fts5ExprCtx sCtx; |
+ sCtx.pExpr = pExpr; |
+ sCtx.aPopulator = aPopulator; |
+ sCtx.iOff = (((i64)iCol) << 32) - 1; |
+ |
+ for(i=0; i<pExpr->nPhrase; i++){ |
+ Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; |
+ Fts5Colset *pColset = pNode->pNear->pColset; |
+ if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) |
+ || aPopulator[i].bMiss |
+ ){ |
+ aPopulator[i].bOk = 0; |
+ }else{ |
+ aPopulator[i].bOk = 1; |
+ } |
+ } |
+ |
+ return sqlite3Fts5Tokenize(pConfig, |
+ FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb |
+ ); |
+} |
+ |
+static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ |
+ if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){ |
+ pNode->pNear->apPhrase[0]->poslist.n = 0; |
+ }else{ |
+ int i; |
+ for(i=0; i<pNode->nChild; i++){ |
+ fts5ExprClearPoslists(pNode->apChild[i]); |
+ } |
+ } |
+} |
+ |
+static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ |
+ pNode->iRowid = iRowid; |
+ pNode->bEof = 0; |
+ switch( pNode->eType ){ |
+ case FTS5_TERM: |
+ case FTS5_STRING: |
+ return (pNode->pNear->apPhrase[0]->poslist.n>0); |
+ |
+ case FTS5_AND: { |
+ int i; |
+ for(i=0; i<pNode->nChild; i++){ |
+ if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ |
+ fts5ExprClearPoslists(pNode); |
+ return 0; |
+ } |
+ } |
+ break; |
+ } |
+ |
+ case FTS5_OR: { |
+ int i; |
+ int bRet = 0; |
+ for(i=0; i<pNode->nChild; i++){ |
+ if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ |
+ bRet = 1; |
+ } |
+ } |
+ return bRet; |
+ } |
+ |
+ default: { |
+ assert( pNode->eType==FTS5_NOT ); |
+ if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) |
+ || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) |
+ ){ |
+ fts5ExprClearPoslists(pNode); |
+ return 0; |
+ } |
+ break; |
+ } |
+ } |
+ return 1; |
+} |
+ |
+static void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ |
+ fts5ExprCheckPoslists(pExpr->pRoot, iRowid); |
+} |
+ |
+/* |
+** This function is only called for detail=columns tables. |
+*/ |
+static int sqlite3Fts5ExprPhraseCollist( |
+ Fts5Expr *pExpr, |
+ int iPhrase, |
+ const u8 **ppCollist, |
+ int *pnCollist |
+){ |
+ Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; |
+ Fts5ExprNode *pNode = pPhrase->pNode; |
+ int rc = SQLITE_OK; |
+ |
+ assert( iPhrase>=0 && iPhrase<pExpr->nPhrase ); |
+ assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); |
+ |
+ if( pNode->bEof==0 |
+ && pNode->iRowid==pExpr->pRoot->iRowid |
+ && pPhrase->poslist.n>0 |
+ ){ |
+ Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; |
+ if( pTerm->pSynonym ){ |
+ Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1]; |
+ rc = fts5ExprSynonymList( |
+ pTerm, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist |
+ ); |
+ }else{ |
+ *ppCollist = pPhrase->aTerm[0].pIter->pData; |
+ *pnCollist = pPhrase->aTerm[0].pIter->nData; |
+ } |
+ }else{ |
+ *ppCollist = 0; |
+ *pnCollist = 0; |
+ } |
+ |
+ return rc; |
+} |
+ |
+ |
+/* |
+** 2014 August 11 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+*/ |
+ |
+ |
+ |
+/* #include "fts5Int.h" */ |
+ |
+typedef struct Fts5HashEntry Fts5HashEntry; |
+ |
+/* |
+** This file contains the implementation of an in-memory hash table used |
+** to accumuluate "term -> doclist" content before it is flused to a level-0 |
+** segment. |
+*/ |
+ |
+ |
+struct Fts5Hash { |
+ int eDetail; /* Copy of Fts5Config.eDetail */ |
+ int *pnByte; /* Pointer to bytes counter */ |
+ int nEntry; /* Number of entries currently in hash */ |
+ int nSlot; /* Size of aSlot[] array */ |
+ Fts5HashEntry *pScan; /* Current ordered scan item */ |
+ Fts5HashEntry **aSlot; /* Array of hash slots */ |
+}; |
+ |
+/* |
+** Each entry in the hash table is represented by an object of the |
+** following type. Each object, its key (zKey[]) and its current data |
+** are stored in a single memory allocation. The position list data |
+** immediately follows the key data in memory. |
+** |
+** The data that follows the key is in a similar, but not identical format |
+** to the doclist data stored in the database. It is: |
+** |
+** * Rowid, as a varint |
+** * Position list, without 0x00 terminator. |
+** * Size of previous position list and rowid, as a 4 byte |
+** big-endian integer. |
+** |
+** iRowidOff: |
+** Offset of last rowid written to data area. Relative to first byte of |
+** structure. |
+** |
+** nData: |
+** Bytes of data written since iRowidOff. |
+*/ |
+struct Fts5HashEntry { |
+ Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */ |
+ Fts5HashEntry *pScanNext; /* Next entry in sorted order */ |
+ |
+ int nAlloc; /* Total size of allocation */ |
+ int iSzPoslist; /* Offset of space for 4-byte poslist size */ |
+ int nData; /* Total bytes of data (incl. structure) */ |
+ int nKey; /* Length of zKey[] in bytes */ |
+ u8 bDel; /* Set delete-flag @ iSzPoslist */ |
+ u8 bContent; /* Set content-flag (detail=none mode) */ |
+ i16 iCol; /* Column of last value written */ |
+ int iPos; /* Position of last value written */ |
+ i64 iRowid; /* Rowid of last value written */ |
+ char zKey[8]; /* Nul-terminated entry key */ |
+}; |
+ |
+/* |
+** Size of Fts5HashEntry without the zKey[] array. |
+*/ |
+#define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8) |
+ |
+ |
+ |
+/* |
+** Allocate a new hash table. |
+*/ |
+static int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ |
+ int rc = SQLITE_OK; |
+ Fts5Hash *pNew; |
+ |
+ *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); |
+ if( pNew==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ int nByte; |
+ memset(pNew, 0, sizeof(Fts5Hash)); |
+ pNew->pnByte = pnByte; |
+ pNew->eDetail = pConfig->eDetail; |
+ |
+ pNew->nSlot = 1024; |
+ nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; |
+ pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte); |
+ if( pNew->aSlot==0 ){ |
+ sqlite3_free(pNew); |
+ *ppNew = 0; |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ memset(pNew->aSlot, 0, nByte); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Free a hash table object. |
+*/ |
+static void sqlite3Fts5HashFree(Fts5Hash *pHash){ |
+ if( pHash ){ |
+ sqlite3Fts5HashClear(pHash); |
+ sqlite3_free(pHash->aSlot); |
+ sqlite3_free(pHash); |
+ } |
+} |
+ |
+/* |
+** Empty (but do not delete) a hash table. |
+*/ |
+static void sqlite3Fts5HashClear(Fts5Hash *pHash){ |
+ int i; |
+ for(i=0; i<pHash->nSlot; i++){ |
+ Fts5HashEntry *pNext; |
+ Fts5HashEntry *pSlot; |
+ for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){ |
+ pNext = pSlot->pHashNext; |
+ sqlite3_free(pSlot); |
+ } |
+ } |
+ memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*)); |
+ pHash->nEntry = 0; |
+} |
+ |
+static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){ |
+ int i; |
+ unsigned int h = 13; |
+ for(i=n-1; i>=0; i--){ |
+ h = (h << 3) ^ h ^ p[i]; |
+ } |
+ return (h % nSlot); |
+} |
+ |
+static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){ |
+ int i; |
+ unsigned int h = 13; |
+ for(i=n-1; i>=0; i--){ |
+ h = (h << 3) ^ h ^ p[i]; |
+ } |
+ h = (h << 3) ^ h ^ b; |
+ return (h % nSlot); |
+} |
+ |
+/* |
+** Resize the hash table by doubling the number of slots. |
+*/ |
+static int fts5HashResize(Fts5Hash *pHash){ |
+ int nNew = pHash->nSlot*2; |
+ int i; |
+ Fts5HashEntry **apNew; |
+ Fts5HashEntry **apOld = pHash->aSlot; |
+ |
+ apNew = (Fts5HashEntry**)sqlite3_malloc(nNew*sizeof(Fts5HashEntry*)); |
+ if( !apNew ) return SQLITE_NOMEM; |
+ memset(apNew, 0, nNew*sizeof(Fts5HashEntry*)); |
+ |
+ for(i=0; i<pHash->nSlot; i++){ |
+ while( apOld[i] ){ |
+ int iHash; |
+ Fts5HashEntry *p = apOld[i]; |
+ apOld[i] = p->pHashNext; |
+ iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey)); |
+ p->pHashNext = apNew[iHash]; |
+ apNew[iHash] = p; |
+ } |
+ } |
+ |
+ sqlite3_free(apOld); |
+ pHash->nSlot = nNew; |
+ pHash->aSlot = apNew; |
+ return SQLITE_OK; |
+} |
+ |
+static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){ |
+ if( p->iSzPoslist ){ |
+ u8 *pPtr = (u8*)p; |
+ if( pHash->eDetail==FTS5_DETAIL_NONE ){ |
+ assert( p->nData==p->iSzPoslist ); |
+ if( p->bDel ){ |
+ pPtr[p->nData++] = 0x00; |
+ if( p->bContent ){ |
+ pPtr[p->nData++] = 0x00; |
+ } |
+ } |
+ }else{ |
+ int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ |
+ int nPos = nSz*2 + p->bDel; /* Value of nPos field */ |
+ |
+ assert( p->bDel==0 || p->bDel==1 ); |
+ if( nPos<=127 ){ |
+ pPtr[p->iSzPoslist] = (u8)nPos; |
+ }else{ |
+ int nByte = sqlite3Fts5GetVarintLen((u32)nPos); |
+ memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); |
+ sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); |
+ p->nData += (nByte-1); |
+ } |
+ } |
+ |
+ p->iSzPoslist = 0; |
+ p->bDel = 0; |
+ p->bContent = 0; |
+ } |
+} |
+ |
+/* |
+** Add an entry to the in-memory hash table. The key is the concatenation |
+** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos). |
+** |
+** (bByte || pToken) -> (iRowid,iCol,iPos) |
+** |
+** Or, if iCol is negative, then the value is a delete marker. |
+*/ |
+static int sqlite3Fts5HashWrite( |
+ Fts5Hash *pHash, |
+ i64 iRowid, /* Rowid for this entry */ |
+ int iCol, /* Column token appears in (-ve -> delete) */ |
+ int iPos, /* Position of token within column */ |
+ char bByte, /* First byte of token */ |
+ const char *pToken, int nToken /* Token to add or remove to or from index */ |
+){ |
+ unsigned int iHash; |
+ Fts5HashEntry *p; |
+ u8 *pPtr; |
+ int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ |
+ int bNew; /* If non-delete entry should be written */ |
+ |
+ bNew = (pHash->eDetail==FTS5_DETAIL_FULL); |
+ |
+ /* Attempt to locate an existing hash entry */ |
+ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); |
+ for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ |
+ if( p->zKey[0]==bByte |
+ && p->nKey==nToken |
+ && memcmp(&p->zKey[1], pToken, nToken)==0 |
+ ){ |
+ break; |
+ } |
+ } |
+ |
+ /* If an existing hash entry cannot be found, create a new one. */ |
+ if( p==0 ){ |
+ /* Figure out how much space to allocate */ |
+ int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; |
+ if( nByte<128 ) nByte = 128; |
+ |
+ /* Grow the Fts5Hash.aSlot[] array if necessary. */ |
+ if( (pHash->nEntry*2)>=pHash->nSlot ){ |
+ int rc = fts5HashResize(pHash); |
+ if( rc!=SQLITE_OK ) return rc; |
+ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); |
+ } |
+ |
+ /* Allocate new Fts5HashEntry and add it to the hash table. */ |
+ p = (Fts5HashEntry*)sqlite3_malloc(nByte); |
+ if( !p ) return SQLITE_NOMEM; |
+ memset(p, 0, FTS5_HASHENTRYSIZE); |
+ p->nAlloc = nByte; |
+ p->zKey[0] = bByte; |
+ memcpy(&p->zKey[1], pToken, nToken); |
+ assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); |
+ p->nKey = nToken; |
+ p->zKey[nToken+1] = '\0'; |
+ p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; |
+ p->pHashNext = pHash->aSlot[iHash]; |
+ pHash->aSlot[iHash] = p; |
+ pHash->nEntry++; |
+ |
+ /* Add the first rowid field to the hash-entry */ |
+ p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); |
+ p->iRowid = iRowid; |
+ |
+ p->iSzPoslist = p->nData; |
+ if( pHash->eDetail!=FTS5_DETAIL_NONE ){ |
+ p->nData += 1; |
+ p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); |
+ } |
+ |
+ nIncr += p->nData; |
+ }else{ |
+ |
+ /* Appending to an existing hash-entry. Check that there is enough |
+ ** space to append the largest possible new entry. Worst case scenario |
+ ** is: |
+ ** |
+ ** + 9 bytes for a new rowid, |
+ ** + 4 byte reserved for the "poslist size" varint. |
+ ** + 1 byte for a "new column" byte, |
+ ** + 3 bytes for a new column number (16-bit max) as a varint, |
+ ** + 5 bytes for the new position offset (32-bit max). |
+ */ |
+ if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ |
+ int nNew = p->nAlloc * 2; |
+ Fts5HashEntry *pNew; |
+ Fts5HashEntry **pp; |
+ pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); |
+ if( pNew==0 ) return SQLITE_NOMEM; |
+ pNew->nAlloc = nNew; |
+ for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); |
+ *pp = pNew; |
+ p = pNew; |
+ } |
+ nIncr -= p->nData; |
+ } |
+ assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) ); |
+ |
+ pPtr = (u8*)p; |
+ |
+ /* If this is a new rowid, append the 4-byte size field for the previous |
+ ** entry, and the new rowid for this entry. */ |
+ if( iRowid!=p->iRowid ){ |
+ fts5HashAddPoslistSize(pHash, p); |
+ p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); |
+ p->iRowid = iRowid; |
+ bNew = 1; |
+ p->iSzPoslist = p->nData; |
+ if( pHash->eDetail!=FTS5_DETAIL_NONE ){ |
+ p->nData += 1; |
+ p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); |
+ p->iPos = 0; |
+ } |
+ } |
+ |
+ if( iCol>=0 ){ |
+ if( pHash->eDetail==FTS5_DETAIL_NONE ){ |
+ p->bContent = 1; |
+ }else{ |
+ /* Append a new column value, if necessary */ |
+ assert( iCol>=p->iCol ); |
+ if( iCol!=p->iCol ){ |
+ if( pHash->eDetail==FTS5_DETAIL_FULL ){ |
+ pPtr[p->nData++] = 0x01; |
+ p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); |
+ p->iCol = (i16)iCol; |
+ p->iPos = 0; |
+ }else{ |
+ bNew = 1; |
+ p->iCol = (i16)(iPos = iCol); |
+ } |
+ } |
+ |
+ /* Append the new position offset, if necessary */ |
+ if( bNew ){ |
+ p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); |
+ p->iPos = iPos; |
+ } |
+ } |
+ }else{ |
+ /* This is a delete. Set the delete flag. */ |
+ p->bDel = 1; |
+ } |
+ |
+ nIncr += p->nData; |
+ *pHash->pnByte += nIncr; |
+ return SQLITE_OK; |
+} |
+ |
+ |
+/* |
+** Arguments pLeft and pRight point to linked-lists of hash-entry objects, |
+** each sorted in key order. This function merges the two lists into a |
+** single list and returns a pointer to its first element. |
+*/ |
+static Fts5HashEntry *fts5HashEntryMerge( |
+ Fts5HashEntry *pLeft, |
+ Fts5HashEntry *pRight |
+){ |
+ Fts5HashEntry *p1 = pLeft; |
+ Fts5HashEntry *p2 = pRight; |
+ Fts5HashEntry *pRet = 0; |
+ Fts5HashEntry **ppOut = &pRet; |
+ |
+ while( p1 || p2 ){ |
+ if( p1==0 ){ |
+ *ppOut = p2; |
+ p2 = 0; |
+ }else if( p2==0 ){ |
+ *ppOut = p1; |
+ p1 = 0; |
+ }else{ |
+ int i = 0; |
+ while( p1->zKey[i]==p2->zKey[i] ) i++; |
+ |
+ if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){ |
+ /* p2 is smaller */ |
+ *ppOut = p2; |
+ ppOut = &p2->pScanNext; |
+ p2 = p2->pScanNext; |
+ }else{ |
+ /* p1 is smaller */ |
+ *ppOut = p1; |
+ ppOut = &p1->pScanNext; |
+ p1 = p1->pScanNext; |
+ } |
+ *ppOut = 0; |
+ } |
+ } |
+ |
+ return pRet; |
+} |
+ |
+/* |
+** Extract all tokens from hash table iHash and link them into a list |
+** in sorted order. The hash table is cleared before returning. It is |
+** the responsibility of the caller to free the elements of the returned |
+** list. |
+*/ |
+static int fts5HashEntrySort( |
+ Fts5Hash *pHash, |
+ const char *pTerm, int nTerm, /* Query prefix, if any */ |
+ Fts5HashEntry **ppSorted |
+){ |
+ const int nMergeSlot = 32; |
+ Fts5HashEntry **ap; |
+ Fts5HashEntry *pList; |
+ int iSlot; |
+ int i; |
+ |
+ *ppSorted = 0; |
+ ap = sqlite3_malloc(sizeof(Fts5HashEntry*) * nMergeSlot); |
+ if( !ap ) return SQLITE_NOMEM; |
+ memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot); |
+ |
+ for(iSlot=0; iSlot<pHash->nSlot; iSlot++){ |
+ Fts5HashEntry *pIter; |
+ for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ |
+ if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){ |
+ Fts5HashEntry *pEntry = pIter; |
+ pEntry->pScanNext = 0; |
+ for(i=0; ap[i]; i++){ |
+ pEntry = fts5HashEntryMerge(pEntry, ap[i]); |
+ ap[i] = 0; |
+ } |
+ ap[i] = pEntry; |
+ } |
+ } |
+ } |
+ |
+ pList = 0; |
+ for(i=0; i<nMergeSlot; i++){ |
+ pList = fts5HashEntryMerge(pList, ap[i]); |
+ } |
+ |
+ pHash->nEntry = 0; |
+ sqlite3_free(ap); |
+ *ppSorted = pList; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Query the hash table for a doclist associated with term pTerm/nTerm. |
+*/ |
+static int sqlite3Fts5HashQuery( |
+ Fts5Hash *pHash, /* Hash table to query */ |
+ const char *pTerm, int nTerm, /* Query term */ |
+ const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ |
+ int *pnDoclist /* OUT: Size of doclist in bytes */ |
+){ |
+ unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm); |
+ Fts5HashEntry *p; |
+ |
+ for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ |
+ if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; |
+ } |
+ |
+ if( p ){ |
+ fts5HashAddPoslistSize(pHash, p); |
+ *ppDoclist = (const u8*)&p->zKey[nTerm+1]; |
+ *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); |
+ }else{ |
+ *ppDoclist = 0; |
+ *pnDoclist = 0; |
+ } |
+ |
+ return SQLITE_OK; |
+} |
+ |
+static int sqlite3Fts5HashScanInit( |
+ Fts5Hash *p, /* Hash table to query */ |
+ const char *pTerm, int nTerm /* Query prefix */ |
+){ |
+ return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan); |
+} |
+ |
+static void sqlite3Fts5HashScanNext(Fts5Hash *p){ |
+ assert( !sqlite3Fts5HashScanEof(p) ); |
+ p->pScan = p->pScan->pScanNext; |
+} |
+ |
+static int sqlite3Fts5HashScanEof(Fts5Hash *p){ |
+ return (p->pScan==0); |
+} |
+ |
+static void sqlite3Fts5HashScanEntry( |
+ Fts5Hash *pHash, |
+ const char **pzTerm, /* OUT: term (nul-terminated) */ |
+ const u8 **ppDoclist, /* OUT: pointer to doclist */ |
+ int *pnDoclist /* OUT: size of doclist in bytes */ |
+){ |
+ Fts5HashEntry *p; |
+ if( (p = pHash->pScan) ){ |
+ int nTerm = (int)strlen(p->zKey); |
+ fts5HashAddPoslistSize(pHash, p); |
+ *pzTerm = p->zKey; |
+ *ppDoclist = (const u8*)&p->zKey[nTerm+1]; |
+ *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); |
+ }else{ |
+ *pzTerm = 0; |
+ *ppDoclist = 0; |
+ *pnDoclist = 0; |
+ } |
+} |
+ |
+ |
+/* |
+** 2014 May 31 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** Low level access to the FTS index stored in the database file. The |
+** routines in this file file implement all read and write access to the |
+** %_data table. Other parts of the system access this functionality via |
+** the interface defined in fts5Int.h. |
+*/ |
+ |
+ |
+/* #include "fts5Int.h" */ |
+ |
+/* |
+** Overview: |
+** |
+** The %_data table contains all the FTS indexes for an FTS5 virtual table. |
+** As well as the main term index, there may be up to 31 prefix indexes. |
+** The format is similar to FTS3/4, except that: |
+** |
+** * all segment b-tree leaf data is stored in fixed size page records |
+** (e.g. 1000 bytes). A single doclist may span multiple pages. Care is |
+** taken to ensure it is possible to iterate in either direction through |
+** the entries in a doclist, or to seek to a specific entry within a |
+** doclist, without loading it into memory. |
+** |
+** * large doclists that span many pages have associated "doclist index" |
+** records that contain a copy of the first rowid on each page spanned by |
+** the doclist. This is used to speed up seek operations, and merges of |
+** large doclists with very small doclists. |
+** |
+** * extra fields in the "structure record" record the state of ongoing |
+** incremental merge operations. |
+** |
+*/ |
+ |
+ |
+#define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ |
+#define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ |
+ |
+#define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ |
+ |
+#define FTS5_MAIN_PREFIX '0' |
+ |
+#if FTS5_MAX_PREFIX_INDEXES > 31 |
+# error "FTS5_MAX_PREFIX_INDEXES is too large" |
+#endif |
+ |
+/* |
+** Details: |
+** |
+** The %_data table managed by this module, |
+** |
+** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); |
+** |
+** , contains the following 5 types of records. See the comments surrounding |
+** the FTS5_*_ROWID macros below for a description of how %_data rowids are |
+** assigned to each fo them. |
+** |
+** 1. Structure Records: |
+** |
+** The set of segments that make up an index - the index structure - are |
+** recorded in a single record within the %_data table. The record consists |
+** of a single 32-bit configuration cookie value followed by a list of |
+** SQLite varints. If the FTS table features more than one index (because |
+** there are one or more prefix indexes), it is guaranteed that all share |
+** the same cookie value. |
+** |
+** Immediately following the configuration cookie, the record begins with |
+** three varints: |
+** |
+** + number of levels, |
+** + total number of segments on all levels, |
+** + value of write counter. |
+** |
+** Then, for each level from 0 to nMax: |
+** |
+** + number of input segments in ongoing merge. |
+** + total number of segments in level. |
+** + for each segment from oldest to newest: |
+** + segment id (always > 0) |
+** + first leaf page number (often 1, always greater than 0) |
+** + final leaf page number |
+** |
+** 2. The Averages Record: |
+** |
+** A single record within the %_data table. The data is a list of varints. |
+** The first value is the number of rows in the index. Then, for each column |
+** from left to right, the total number of tokens in the column for all |
+** rows of the table. |
+** |
+** 3. Segment leaves: |
+** |
+** TERM/DOCLIST FORMAT: |
+** |
+** Most of each segment leaf is taken up by term/doclist data. The |
+** general format of term/doclist, starting with the first term |
+** on the leaf page, is: |
+** |
+** varint : size of first term |
+** blob: first term data |
+** doclist: first doclist |
+** zero-or-more { |
+** varint: number of bytes in common with previous term |
+** varint: number of bytes of new term data (nNew) |
+** blob: nNew bytes of new term data |
+** doclist: next doclist |
+** } |
+** |
+** doclist format: |
+** |
+** varint: first rowid |
+** poslist: first poslist |
+** zero-or-more { |
+** varint: rowid delta (always > 0) |
+** poslist: next poslist |
+** } |
+** |
+** poslist format: |
+** |
+** varint: size of poslist in bytes multiplied by 2, not including |
+** this field. Plus 1 if this entry carries the "delete" flag. |
+** collist: collist for column 0 |
+** zero-or-more { |
+** 0x01 byte |
+** varint: column number (I) |
+** collist: collist for column I |
+** } |
+** |
+** collist format: |
+** |
+** varint: first offset + 2 |
+** zero-or-more { |
+** varint: offset delta + 2 |
+** } |
+** |
+** PAGE FORMAT |
+** |
+** Each leaf page begins with a 4-byte header containing 2 16-bit |
+** unsigned integer fields in big-endian format. They are: |
+** |
+** * The byte offset of the first rowid on the page, if it exists |
+** and occurs before the first term (otherwise 0). |
+** |
+** * The byte offset of the start of the page footer. If the page |
+** footer is 0 bytes in size, then this field is the same as the |
+** size of the leaf page in bytes. |
+** |
+** The page footer consists of a single varint for each term located |
+** on the page. Each varint is the byte offset of the current term |
+** within the page, delta-compressed against the previous value. In |
+** other words, the first varint in the footer is the byte offset of |
+** the first term, the second is the byte offset of the second less that |
+** of the first, and so on. |
+** |
+** The term/doclist format described above is accurate if the entire |
+** term/doclist data fits on a single leaf page. If this is not the case, |
+** the format is changed in two ways: |
+** |
+** + if the first rowid on a page occurs before the first term, it |
+** is stored as a literal value: |
+** |
+** varint: first rowid |
+** |
+** + the first term on each page is stored in the same way as the |
+** very first term of the segment: |
+** |
+** varint : size of first term |
+** blob: first term data |
+** |
+** 5. Segment doclist indexes: |
+** |
+** Doclist indexes are themselves b-trees, however they usually consist of |
+** a single leaf record only. The format of each doclist index leaf page |
+** is: |
+** |
+** * Flags byte. Bits are: |
+** 0x01: Clear if leaf is also the root page, otherwise set. |
+** |
+** * Page number of fts index leaf page. As a varint. |
+** |
+** * First rowid on page indicated by previous field. As a varint. |
+** |
+** * A list of varints, one for each subsequent termless page. A |
+** positive delta if the termless page contains at least one rowid, |
+** or an 0x00 byte otherwise. |
+** |
+** Internal doclist index nodes are: |
+** |
+** * Flags byte. Bits are: |
+** 0x01: Clear for root page, otherwise set. |
+** |
+** * Page number of first child page. As a varint. |
+** |
+** * Copy of first rowid on page indicated by previous field. As a varint. |
+** |
+** * A list of delta-encoded varints - the first rowid on each subsequent |
+** child page. |
+** |
+*/ |
+ |
+/* |
+** Rowids for the averages and structure records in the %_data table. |
+*/ |
+#define FTS5_AVERAGES_ROWID 1 /* Rowid used for the averages record */ |
+#define FTS5_STRUCTURE_ROWID 10 /* The structure record */ |
+ |
+/* |
+** Macros determining the rowids used by segment leaves and dlidx leaves |
+** and nodes. All nodes and leaves are stored in the %_data table with large |
+** positive rowids. |
+** |
+** Each segment has a unique non-zero 16-bit id. |
+** |
+** The rowid for each segment leaf is found by passing the segment id and |
+** the leaf page number to the FTS5_SEGMENT_ROWID macro. Leaves are numbered |
+** sequentially starting from 1. |
+*/ |
+#define FTS5_DATA_ID_B 16 /* Max seg id number 65535 */ |
+#define FTS5_DATA_DLI_B 1 /* Doclist-index flag (1 bit) */ |
+#define FTS5_DATA_HEIGHT_B 5 /* Max dlidx tree height of 32 */ |
+#define FTS5_DATA_PAGE_B 31 /* Max page number of 2147483648 */ |
+ |
+#define fts5_dri(segid, dlidx, height, pgno) ( \ |
+ ((i64)(segid) << (FTS5_DATA_PAGE_B+FTS5_DATA_HEIGHT_B+FTS5_DATA_DLI_B)) + \ |
+ ((i64)(dlidx) << (FTS5_DATA_PAGE_B + FTS5_DATA_HEIGHT_B)) + \ |
+ ((i64)(height) << (FTS5_DATA_PAGE_B)) + \ |
+ ((i64)(pgno)) \ |
+) |
+ |
+#define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno) |
+#define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno) |
+ |
+/* |
+** Maximum segments permitted in a single index |
+*/ |
+#define FTS5_MAX_SEGMENT 2000 |
+ |
+#ifdef SQLITE_DEBUG |
+static int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } |
+#endif |
+ |
+ |
+/* |
+** Each time a blob is read from the %_data table, it is padded with this |
+** many zero bytes. This makes it easier to decode the various record formats |
+** without overreading if the records are corrupt. |
+*/ |
+#define FTS5_DATA_ZERO_PADDING 8 |
+#define FTS5_DATA_PADDING 20 |
+ |
+typedef struct Fts5Data Fts5Data; |
+typedef struct Fts5DlidxIter Fts5DlidxIter; |
+typedef struct Fts5DlidxLvl Fts5DlidxLvl; |
+typedef struct Fts5DlidxWriter Fts5DlidxWriter; |
+typedef struct Fts5Iter Fts5Iter; |
+typedef struct Fts5PageWriter Fts5PageWriter; |
+typedef struct Fts5SegIter Fts5SegIter; |
+typedef struct Fts5DoclistIter Fts5DoclistIter; |
+typedef struct Fts5SegWriter Fts5SegWriter; |
+typedef struct Fts5Structure Fts5Structure; |
+typedef struct Fts5StructureLevel Fts5StructureLevel; |
+typedef struct Fts5StructureSegment Fts5StructureSegment; |
+ |
+struct Fts5Data { |
+ u8 *p; /* Pointer to buffer containing record */ |
+ int nn; /* Size of record in bytes */ |
+ int szLeaf; /* Size of leaf without page-index */ |
+}; |
+ |
+/* |
+** One object per %_data table. |
+*/ |
+struct Fts5Index { |
+ Fts5Config *pConfig; /* Virtual table configuration */ |
+ char *zDataTbl; /* Name of %_data table */ |
+ int nWorkUnit; /* Leaf pages in a "unit" of work */ |
+ |
+ /* |
+ ** Variables related to the accumulation of tokens and doclists within the |
+ ** in-memory hash tables before they are flushed to disk. |
+ */ |
+ Fts5Hash *pHash; /* Hash table for in-memory data */ |
+ int nPendingData; /* Current bytes of pending data */ |
+ i64 iWriteRowid; /* Rowid for current doc being written */ |
+ int bDelete; /* Current write is a delete */ |
+ |
+ /* Error state. */ |
+ int rc; /* Current error code */ |
+ |
+ /* State used by the fts5DataXXX() functions. */ |
+ sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ |
+ sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ |
+ sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ |
+ sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ |
+ sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ |
+ sqlite3_stmt *pIdxSelect; |
+ int nRead; /* Total number of blocks read */ |
+ |
+ sqlite3_stmt *pDataVersion; |
+ i64 iStructVersion; /* data_version when pStruct read */ |
+ Fts5Structure *pStruct; /* Current db structure (or NULL) */ |
+}; |
+ |
+struct Fts5DoclistIter { |
+ u8 *aEof; /* Pointer to 1 byte past end of doclist */ |
+ |
+ /* Output variables. aPoslist==0 at EOF */ |
+ i64 iRowid; |
+ u8 *aPoslist; |
+ int nPoslist; |
+ int nSize; |
+}; |
+ |
+/* |
+** The contents of the "structure" record for each index are represented |
+** using an Fts5Structure record in memory. Which uses instances of the |
+** other Fts5StructureXXX types as components. |
+*/ |
+struct Fts5StructureSegment { |
+ int iSegid; /* Segment id */ |
+ int pgnoFirst; /* First leaf page number in segment */ |
+ int pgnoLast; /* Last leaf page number in segment */ |
+}; |
+struct Fts5StructureLevel { |
+ int nMerge; /* Number of segments in incr-merge */ |
+ int nSeg; /* Total number of segments on level */ |
+ Fts5StructureSegment *aSeg; /* Array of segments. aSeg[0] is oldest. */ |
+}; |
+struct Fts5Structure { |
+ int nRef; /* Object reference count */ |
+ u64 nWriteCounter; /* Total leaves written to level 0 */ |
+ int nSegment; /* Total segments in this structure */ |
+ int nLevel; /* Number of levels in this index */ |
+ Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ |
+}; |
+ |
+/* |
+** An object of type Fts5SegWriter is used to write to segments. |
+*/ |
+struct Fts5PageWriter { |
+ int pgno; /* Page number for this page */ |
+ int iPrevPgidx; /* Previous value written into pgidx */ |
+ Fts5Buffer buf; /* Buffer containing leaf data */ |
+ Fts5Buffer pgidx; /* Buffer containing page-index */ |
+ Fts5Buffer term; /* Buffer containing previous term on page */ |
+}; |
+struct Fts5DlidxWriter { |
+ int pgno; /* Page number for this page */ |
+ int bPrevValid; /* True if iPrev is valid */ |
+ i64 iPrev; /* Previous rowid value written to page */ |
+ Fts5Buffer buf; /* Buffer containing page data */ |
+}; |
+struct Fts5SegWriter { |
+ int iSegid; /* Segid to write to */ |
+ Fts5PageWriter writer; /* PageWriter object */ |
+ i64 iPrevRowid; /* Previous rowid written to current leaf */ |
+ u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */ |
+ u8 bFirstRowidInPage; /* True if next rowid is first in page */ |
+ /* TODO1: Can use (writer.pgidx.n==0) instead of bFirstTermInPage */ |
+ u8 bFirstTermInPage; /* True if next term will be first in leaf */ |
+ int nLeafWritten; /* Number of leaf pages written */ |
+ int nEmpty; /* Number of contiguous term-less nodes */ |
+ |
+ int nDlidx; /* Allocated size of aDlidx[] array */ |
+ Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */ |
+ |
+ /* Values to insert into the %_idx table */ |
+ Fts5Buffer btterm; /* Next term to insert into %_idx table */ |
+ int iBtPage; /* Page number corresponding to btterm */ |
+}; |
+ |
+typedef struct Fts5CResult Fts5CResult; |
+struct Fts5CResult { |
+ u16 iFirst; /* aSeg[] index of firstest iterator */ |
+ u8 bTermEq; /* True if the terms are equal */ |
+}; |
+ |
+/* |
+** Object for iterating through a single segment, visiting each term/rowid |
+** pair in the segment. |
+** |
+** pSeg: |
+** The segment to iterate through. |
+** |
+** iLeafPgno: |
+** Current leaf page number within segment. |
+** |
+** iLeafOffset: |
+** Byte offset within the current leaf that is the first byte of the |
+** position list data (one byte passed the position-list size field). |
+** rowid field of the current entry. Usually this is the size field of the |
+** position list data. The exception is if the rowid for the current entry |
+** is the last thing on the leaf page. |
+** |
+** pLeaf: |
+** Buffer containing current leaf page data. Set to NULL at EOF. |
+** |
+** iTermLeafPgno, iTermLeafOffset: |
+** Leaf page number containing the last term read from the segment. And |
+** the offset immediately following the term data. |
+** |
+** flags: |
+** Mask of FTS5_SEGITER_XXX values. Interpreted as follows: |
+** |
+** FTS5_SEGITER_ONETERM: |
+** If set, set the iterator to point to EOF after the current doclist |
+** has been exhausted. Do not proceed to the next term in the segment. |
+** |
+** FTS5_SEGITER_REVERSE: |
+** This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If |
+** it is set, iterate through rowid in descending order instead of the |
+** default ascending order. |
+** |
+** iRowidOffset/nRowidOffset/aRowidOffset: |
+** These are used if the FTS5_SEGITER_REVERSE flag is set. |
+** |
+** For each rowid on the page corresponding to the current term, the |
+** corresponding aRowidOffset[] entry is set to the byte offset of the |
+** start of the "position-list-size" field within the page. |
+** |
+** iTermIdx: |
+** Index of current term on iTermLeafPgno. |
+*/ |
+struct Fts5SegIter { |
+ Fts5StructureSegment *pSeg; /* Segment to iterate through */ |
+ int flags; /* Mask of configuration flags */ |
+ int iLeafPgno; /* Current leaf page number */ |
+ Fts5Data *pLeaf; /* Current leaf data */ |
+ Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ |
+ int iLeafOffset; /* Byte offset within current leaf */ |
+ |
+ /* Next method */ |
+ void (*xNext)(Fts5Index*, Fts5SegIter*, int*); |
+ |
+ /* The page and offset from which the current term was read. The offset |
+ ** is the offset of the first rowid in the current doclist. */ |
+ int iTermLeafPgno; |
+ int iTermLeafOffset; |
+ |
+ int iPgidxOff; /* Next offset in pgidx */ |
+ int iEndofDoclist; |
+ |
+ /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */ |
+ int iRowidOffset; /* Current entry in aRowidOffset[] */ |
+ int nRowidOffset; /* Allocated size of aRowidOffset[] array */ |
+ int *aRowidOffset; /* Array of offset to rowid fields */ |
+ |
+ Fts5DlidxIter *pDlidx; /* If there is a doclist-index */ |
+ |
+ /* Variables populated based on current entry. */ |
+ Fts5Buffer term; /* Current term */ |
+ i64 iRowid; /* Current rowid */ |
+ int nPos; /* Number of bytes in current position list */ |
+ u8 bDel; /* True if the delete flag is set */ |
+}; |
+ |
+/* |
+** Argument is a pointer to an Fts5Data structure that contains a |
+** leaf page. |
+*/ |
+#define ASSERT_SZLEAF_OK(x) assert( \ |
+ (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \ |
+) |
+ |
+#define FTS5_SEGITER_ONETERM 0x01 |
+#define FTS5_SEGITER_REVERSE 0x02 |
+ |
+/* |
+** Argument is a pointer to an Fts5Data structure that contains a leaf |
+** page. This macro evaluates to true if the leaf contains no terms, or |
+** false if it contains at least one term. |
+*/ |
+#define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) |
+ |
+#define fts5LeafTermOff(x, i) (fts5GetU16(&(x)->p[(x)->szLeaf + (i)*2])) |
+ |
+#define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p)) |
+ |
+/* |
+** Object for iterating through the merged results of one or more segments, |
+** visiting each term/rowid pair in the merged data. |
+** |
+** nSeg is always a power of two greater than or equal to the number of |
+** segments that this object is merging data from. Both the aSeg[] and |
+** aFirst[] arrays are sized at nSeg entries. The aSeg[] array is padded |
+** with zeroed objects - these are handled as if they were iterators opened |
+** on empty segments. |
+** |
+** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an |
+** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the |
+** comparison in this context is the index of the iterator that currently |
+** points to the smaller term/rowid combination. Iterators at EOF are |
+** considered to be greater than all other iterators. |
+** |
+** aFirst[1] contains the index in aSeg[] of the iterator that points to |
+** the smallest key overall. aFirst[0] is unused. |
+** |
+** poslist: |
+** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. |
+** There is no way to tell if this is populated or not. |
+*/ |
+struct Fts5Iter { |
+ Fts5IndexIter base; /* Base class containing output vars */ |
+ |
+ Fts5Index *pIndex; /* Index that owns this iterator */ |
+ Fts5Structure *pStruct; /* Database structure for this iterator */ |
+ Fts5Buffer poslist; /* Buffer containing current poslist */ |
+ Fts5Colset *pColset; /* Restrict matches to these columns */ |
+ |
+ /* Invoked to set output variables. */ |
+ void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*); |
+ |
+ int nSeg; /* Size of aSeg[] array */ |
+ int bRev; /* True to iterate in reverse order */ |
+ u8 bSkipEmpty; /* True to skip deleted entries */ |
+ |
+ i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ |
+ Fts5CResult *aFirst; /* Current merge state (see above) */ |
+ Fts5SegIter aSeg[1]; /* Array of segment iterators */ |
+}; |
+ |
+ |
+/* |
+** An instance of the following type is used to iterate through the contents |
+** of a doclist-index record. |
+** |
+** pData: |
+** Record containing the doclist-index data. |
+** |
+** bEof: |
+** Set to true once iterator has reached EOF. |
+** |
+** iOff: |
+** Set to the current offset within record pData. |
+*/ |
+struct Fts5DlidxLvl { |
+ Fts5Data *pData; /* Data for current page of this level */ |
+ int iOff; /* Current offset into pData */ |
+ int bEof; /* At EOF already */ |
+ int iFirstOff; /* Used by reverse iterators */ |
+ |
+ /* Output variables */ |
+ int iLeafPgno; /* Page number of current leaf page */ |
+ i64 iRowid; /* First rowid on leaf iLeafPgno */ |
+}; |
+struct Fts5DlidxIter { |
+ int nLvl; |
+ int iSegid; |
+ Fts5DlidxLvl aLvl[1]; |
+}; |
+ |
+static void fts5PutU16(u8 *aOut, u16 iVal){ |
+ aOut[0] = (iVal>>8); |
+ aOut[1] = (iVal&0xFF); |
+} |
+ |
+static u16 fts5GetU16(const u8 *aIn){ |
+ return ((u16)aIn[0] << 8) + aIn[1]; |
+} |
+ |
+/* |
+** Allocate and return a buffer at least nByte bytes in size. |
+** |
+** If an OOM error is encountered, return NULL and set the error code in |
+** the Fts5Index handle passed as the first argument. |
+*/ |
+static void *fts5IdxMalloc(Fts5Index *p, int nByte){ |
+ return sqlite3Fts5MallocZero(&p->rc, nByte); |
+} |
+ |
+/* |
+** Compare the contents of the pLeft buffer with the pRight/nRight blob. |
+** |
+** Return -ve if pLeft is smaller than pRight, 0 if they are equal or |
+** +ve if pRight is smaller than pLeft. In other words: |
+** |
+** res = *pLeft - *pRight |
+*/ |
+#ifdef SQLITE_DEBUG |
+static int fts5BufferCompareBlob( |
+ Fts5Buffer *pLeft, /* Left hand side of comparison */ |
+ const u8 *pRight, int nRight /* Right hand side of comparison */ |
+){ |
+ int nCmp = MIN(pLeft->n, nRight); |
+ int res = memcmp(pLeft->p, pRight, nCmp); |
+ return (res==0 ? (pLeft->n - nRight) : res); |
+} |
+#endif |
+ |
+/* |
+** Compare the contents of the two buffers using memcmp(). If one buffer |
+** is a prefix of the other, it is considered the lesser. |
+** |
+** Return -ve if pLeft is smaller than pRight, 0 if they are equal or |
+** +ve if pRight is smaller than pLeft. In other words: |
+** |
+** res = *pLeft - *pRight |
+*/ |
+static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){ |
+ int nCmp = MIN(pLeft->n, pRight->n); |
+ int res = memcmp(pLeft->p, pRight->p, nCmp); |
+ return (res==0 ? (pLeft->n - pRight->n) : res); |
+} |
+ |
+static int fts5LeafFirstTermOff(Fts5Data *pLeaf){ |
+ int ret; |
+ fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret); |
+ return ret; |
+} |
+ |
+/* |
+** Close the read-only blob handle, if it is open. |
+*/ |
+static void fts5CloseReader(Fts5Index *p){ |
+ if( p->pReader ){ |
+ sqlite3_blob *pReader = p->pReader; |
+ p->pReader = 0; |
+ sqlite3_blob_close(pReader); |
+ } |
+} |
+ |
+ |
+/* |
+** Retrieve a record from the %_data table. |
+** |
+** If an error occurs, NULL is returned and an error left in the |
+** Fts5Index object. |
+*/ |
+static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ |
+ Fts5Data *pRet = 0; |
+ if( p->rc==SQLITE_OK ){ |
+ int rc = SQLITE_OK; |
+ |
+ if( p->pReader ){ |
+ /* This call may return SQLITE_ABORT if there has been a savepoint |
+ ** rollback since it was last used. In this case a new blob handle |
+ ** is required. */ |
+ sqlite3_blob *pBlob = p->pReader; |
+ p->pReader = 0; |
+ rc = sqlite3_blob_reopen(pBlob, iRowid); |
+ assert( p->pReader==0 ); |
+ p->pReader = pBlob; |
+ if( rc!=SQLITE_OK ){ |
+ fts5CloseReader(p); |
+ } |
+ if( rc==SQLITE_ABORT ) rc = SQLITE_OK; |
+ } |
+ |
+ /* If the blob handle is not open at this point, open it and seek |
+ ** to the requested entry. */ |
+ if( p->pReader==0 && rc==SQLITE_OK ){ |
+ Fts5Config *pConfig = p->pConfig; |
+ rc = sqlite3_blob_open(pConfig->db, |
+ pConfig->zDb, p->zDataTbl, "block", iRowid, 0, &p->pReader |
+ ); |
+ } |
+ |
+ /* If either of the sqlite3_blob_open() or sqlite3_blob_reopen() calls |
+ ** above returned SQLITE_ERROR, return SQLITE_CORRUPT_VTAB instead. |
+ ** All the reasons those functions might return SQLITE_ERROR - missing |
+ ** table, missing row, non-blob/text in block column - indicate |
+ ** backing store corruption. */ |
+ if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT; |
+ |
+ if( rc==SQLITE_OK ){ |
+ u8 *aOut = 0; /* Read blob data into this buffer */ |
+ int nByte = sqlite3_blob_bytes(p->pReader); |
+ int nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING; |
+ pRet = (Fts5Data*)sqlite3_malloc(nAlloc); |
+ if( pRet ){ |
+ pRet->nn = nByte; |
+ aOut = pRet->p = (u8*)&pRet[1]; |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3_blob_read(p->pReader, aOut, nByte, 0); |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_free(pRet); |
+ pRet = 0; |
+ }else{ |
+ /* TODO1: Fix this */ |
+ pRet->szLeaf = fts5GetU16(&pRet->p[2]); |
+ } |
+ } |
+ p->rc = rc; |
+ p->nRead++; |
+ } |
+ |
+ assert( (pRet==0)==(p->rc!=SQLITE_OK) ); |
+ return pRet; |
+} |
+ |
+/* |
+** Release a reference to data record returned by an earlier call to |
+** fts5DataRead(). |
+*/ |
+static void fts5DataRelease(Fts5Data *pData){ |
+ sqlite3_free(pData); |
+} |
+ |
+static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ |
+ Fts5Data *pRet = fts5DataRead(p, iRowid); |
+ if( pRet ){ |
+ if( pRet->szLeaf>pRet->nn ){ |
+ p->rc = FTS5_CORRUPT; |
+ fts5DataRelease(pRet); |
+ pRet = 0; |
+ } |
+ } |
+ return pRet; |
+} |
+ |
+static int fts5IndexPrepareStmt( |
+ Fts5Index *p, |
+ sqlite3_stmt **ppStmt, |
+ char *zSql |
+){ |
+ if( p->rc==SQLITE_OK ){ |
+ if( zSql ){ |
+ p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0); |
+ }else{ |
+ p->rc = SQLITE_NOMEM; |
+ } |
+ } |
+ sqlite3_free(zSql); |
+ return p->rc; |
+} |
+ |
+ |
+/* |
+** INSERT OR REPLACE a record into the %_data table. |
+*/ |
+static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ |
+ if( p->rc!=SQLITE_OK ) return; |
+ |
+ if( p->pWriter==0 ){ |
+ Fts5Config *pConfig = p->pConfig; |
+ fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf( |
+ "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)", |
+ pConfig->zDb, pConfig->zName |
+ )); |
+ if( p->rc ) return; |
+ } |
+ |
+ sqlite3_bind_int64(p->pWriter, 1, iRowid); |
+ sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC); |
+ sqlite3_step(p->pWriter); |
+ p->rc = sqlite3_reset(p->pWriter); |
+} |
+ |
+/* |
+** Execute the following SQL: |
+** |
+** DELETE FROM %_data WHERE id BETWEEN $iFirst AND $iLast |
+*/ |
+static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ |
+ if( p->rc!=SQLITE_OK ) return; |
+ |
+ if( p->pDeleter==0 ){ |
+ int rc; |
+ Fts5Config *pConfig = p->pConfig; |
+ char *zSql = sqlite3_mprintf( |
+ "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", |
+ pConfig->zDb, pConfig->zName |
+ ); |
+ if( zSql==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0); |
+ sqlite3_free(zSql); |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ p->rc = rc; |
+ return; |
+ } |
+ } |
+ |
+ sqlite3_bind_int64(p->pDeleter, 1, iFirst); |
+ sqlite3_bind_int64(p->pDeleter, 2, iLast); |
+ sqlite3_step(p->pDeleter); |
+ p->rc = sqlite3_reset(p->pDeleter); |
+} |
+ |
+/* |
+** Remove all records associated with segment iSegid. |
+*/ |
+static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ |
+ i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0); |
+ i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1; |
+ fts5DataDelete(p, iFirst, iLast); |
+ if( p->pIdxDeleter==0 ){ |
+ Fts5Config *pConfig = p->pConfig; |
+ fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( |
+ "DELETE FROM '%q'.'%q_idx' WHERE segid=?", |
+ pConfig->zDb, pConfig->zName |
+ )); |
+ } |
+ if( p->rc==SQLITE_OK ){ |
+ sqlite3_bind_int(p->pIdxDeleter, 1, iSegid); |
+ sqlite3_step(p->pIdxDeleter); |
+ p->rc = sqlite3_reset(p->pIdxDeleter); |
+ } |
+} |
+ |
+/* |
+** Release a reference to an Fts5Structure object returned by an earlier |
+** call to fts5StructureRead() or fts5StructureDecode(). |
+*/ |
+static void fts5StructureRelease(Fts5Structure *pStruct){ |
+ if( pStruct && 0>=(--pStruct->nRef) ){ |
+ int i; |
+ assert( pStruct->nRef==0 ); |
+ for(i=0; i<pStruct->nLevel; i++){ |
+ sqlite3_free(pStruct->aLevel[i].aSeg); |
+ } |
+ sqlite3_free(pStruct); |
+ } |
+} |
+ |
+static void fts5StructureRef(Fts5Structure *pStruct){ |
+ pStruct->nRef++; |
+} |
+ |
+/* |
+** Deserialize and return the structure record currently stored in serialized |
+** form within buffer pData/nData. |
+** |
+** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array |
+** are over-allocated by one slot. This allows the structure contents |
+** to be more easily edited. |
+** |
+** If an error occurs, *ppOut is set to NULL and an SQLite error code |
+** returned. Otherwise, *ppOut is set to point to the new object and |
+** SQLITE_OK returned. |
+*/ |
+static int fts5StructureDecode( |
+ const u8 *pData, /* Buffer containing serialized structure */ |
+ int nData, /* Size of buffer pData in bytes */ |
+ int *piCookie, /* Configuration cookie value */ |
+ Fts5Structure **ppOut /* OUT: Deserialized object */ |
+){ |
+ int rc = SQLITE_OK; |
+ int i = 0; |
+ int iLvl; |
+ int nLevel = 0; |
+ int nSegment = 0; |
+ int nByte; /* Bytes of space to allocate at pRet */ |
+ Fts5Structure *pRet = 0; /* Structure object to return */ |
+ |
+ /* Grab the cookie value */ |
+ if( piCookie ) *piCookie = sqlite3Fts5Get32(pData); |
+ i = 4; |
+ |
+ /* Read the total number of levels and segments from the start of the |
+ ** structure record. */ |
+ i += fts5GetVarint32(&pData[i], nLevel); |
+ i += fts5GetVarint32(&pData[i], nSegment); |
+ nByte = ( |
+ sizeof(Fts5Structure) + /* Main structure */ |
+ sizeof(Fts5StructureLevel) * (nLevel-1) /* aLevel[] array */ |
+ ); |
+ pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); |
+ |
+ if( pRet ){ |
+ pRet->nRef = 1; |
+ pRet->nLevel = nLevel; |
+ pRet->nSegment = nSegment; |
+ i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter); |
+ |
+ for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){ |
+ Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl]; |
+ int nTotal = 0; |
+ int iSeg; |
+ |
+ if( i>=nData ){ |
+ rc = FTS5_CORRUPT; |
+ }else{ |
+ i += fts5GetVarint32(&pData[i], pLvl->nMerge); |
+ i += fts5GetVarint32(&pData[i], nTotal); |
+ assert( nTotal>=pLvl->nMerge ); |
+ pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, |
+ nTotal * sizeof(Fts5StructureSegment) |
+ ); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ pLvl->nSeg = nTotal; |
+ for(iSeg=0; iSeg<nTotal; iSeg++){ |
+ if( i>=nData ){ |
+ rc = FTS5_CORRUPT; |
+ break; |
+ } |
+ i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid); |
+ i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst); |
+ i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast); |
+ } |
+ } |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ fts5StructureRelease(pRet); |
+ pRet = 0; |
+ } |
+ } |
+ |
+ *ppOut = pRet; |
+ return rc; |
+} |
+ |
+/* |
+** |
+*/ |
+static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ |
+ if( *pRc==SQLITE_OK ){ |
+ Fts5Structure *pStruct = *ppStruct; |
+ int nLevel = pStruct->nLevel; |
+ int nByte = ( |
+ sizeof(Fts5Structure) + /* Main structure */ |
+ sizeof(Fts5StructureLevel) * (nLevel+1) /* aLevel[] array */ |
+ ); |
+ |
+ pStruct = sqlite3_realloc(pStruct, nByte); |
+ if( pStruct ){ |
+ memset(&pStruct->aLevel[nLevel], 0, sizeof(Fts5StructureLevel)); |
+ pStruct->nLevel++; |
+ *ppStruct = pStruct; |
+ }else{ |
+ *pRc = SQLITE_NOMEM; |
+ } |
+ } |
+} |
+ |
+/* |
+** Extend level iLvl so that there is room for at least nExtra more |
+** segments. |
+*/ |
+static void fts5StructureExtendLevel( |
+ int *pRc, |
+ Fts5Structure *pStruct, |
+ int iLvl, |
+ int nExtra, |
+ int bInsert |
+){ |
+ if( *pRc==SQLITE_OK ){ |
+ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; |
+ Fts5StructureSegment *aNew; |
+ int nByte; |
+ |
+ nByte = (pLvl->nSeg + nExtra) * sizeof(Fts5StructureSegment); |
+ aNew = sqlite3_realloc(pLvl->aSeg, nByte); |
+ if( aNew ){ |
+ if( bInsert==0 ){ |
+ memset(&aNew[pLvl->nSeg], 0, sizeof(Fts5StructureSegment) * nExtra); |
+ }else{ |
+ int nMove = pLvl->nSeg * sizeof(Fts5StructureSegment); |
+ memmove(&aNew[nExtra], aNew, nMove); |
+ memset(aNew, 0, sizeof(Fts5StructureSegment) * nExtra); |
+ } |
+ pLvl->aSeg = aNew; |
+ }else{ |
+ *pRc = SQLITE_NOMEM; |
+ } |
+ } |
+} |
+ |
+static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ |
+ Fts5Structure *pRet = 0; |
+ Fts5Config *pConfig = p->pConfig; |
+ int iCookie; /* Configuration cookie */ |
+ Fts5Data *pData; |
+ |
+ pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); |
+ if( p->rc==SQLITE_OK ){ |
+ /* TODO: Do we need this if the leaf-index is appended? Probably... */ |
+ memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); |
+ p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); |
+ if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ |
+ p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); |
+ } |
+ fts5DataRelease(pData); |
+ if( p->rc!=SQLITE_OK ){ |
+ fts5StructureRelease(pRet); |
+ pRet = 0; |
+ } |
+ } |
+ |
+ return pRet; |
+} |
+ |
+static i64 fts5IndexDataVersion(Fts5Index *p){ |
+ i64 iVersion = 0; |
+ |
+ if( p->rc==SQLITE_OK ){ |
+ if( p->pDataVersion==0 ){ |
+ p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, |
+ sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb) |
+ ); |
+ if( p->rc ) return 0; |
+ } |
+ |
+ if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){ |
+ iVersion = sqlite3_column_int64(p->pDataVersion, 0); |
+ } |
+ p->rc = sqlite3_reset(p->pDataVersion); |
+ } |
+ |
+ return iVersion; |
+} |
+ |
+/* |
+** Read, deserialize and return the structure record. |
+** |
+** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array |
+** are over-allocated as described for function fts5StructureDecode() |
+** above. |
+** |
+** If an error occurs, NULL is returned and an error code left in the |
+** Fts5Index handle. If an error has already occurred when this function |
+** is called, it is a no-op. |
+*/ |
+static Fts5Structure *fts5StructureRead(Fts5Index *p){ |
+ |
+ if( p->pStruct==0 ){ |
+ p->iStructVersion = fts5IndexDataVersion(p); |
+ if( p->rc==SQLITE_OK ){ |
+ p->pStruct = fts5StructureReadUncached(p); |
+ } |
+ } |
+ |
+#if 0 |
+ else{ |
+ Fts5Structure *pTest = fts5StructureReadUncached(p); |
+ if( pTest ){ |
+ int i, j; |
+ assert_nc( p->pStruct->nSegment==pTest->nSegment ); |
+ assert_nc( p->pStruct->nLevel==pTest->nLevel ); |
+ for(i=0; i<pTest->nLevel; i++){ |
+ assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge ); |
+ assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg ); |
+ for(j=0; j<pTest->aLevel[i].nSeg; j++){ |
+ Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j]; |
+ Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j]; |
+ assert_nc( p1->iSegid==p2->iSegid ); |
+ assert_nc( p1->pgnoFirst==p2->pgnoFirst ); |
+ assert_nc( p1->pgnoLast==p2->pgnoLast ); |
+ } |
+ } |
+ fts5StructureRelease(pTest); |
+ } |
+ } |
+#endif |
+ |
+ if( p->rc!=SQLITE_OK ) return 0; |
+ assert( p->iStructVersion!=0 ); |
+ assert( p->pStruct!=0 ); |
+ fts5StructureRef(p->pStruct); |
+ return p->pStruct; |
+} |
+ |
+static void fts5StructureInvalidate(Fts5Index *p){ |
+ if( p->pStruct ){ |
+ fts5StructureRelease(p->pStruct); |
+ p->pStruct = 0; |
+ } |
+} |
+ |
+/* |
+** Return the total number of segments in index structure pStruct. This |
+** function is only ever used as part of assert() conditions. |
+*/ |
+#ifdef SQLITE_DEBUG |
+static int fts5StructureCountSegments(Fts5Structure *pStruct){ |
+ int nSegment = 0; /* Total number of segments */ |
+ if( pStruct ){ |
+ int iLvl; /* Used to iterate through levels */ |
+ for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ |
+ nSegment += pStruct->aLevel[iLvl].nSeg; |
+ } |
+ } |
+ |
+ return nSegment; |
+} |
+#endif |
+ |
+#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ |
+ assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) ); \ |
+ memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \ |
+ (pBuf)->n += nBlob; \ |
+} |
+ |
+#define fts5BufferSafeAppendVarint(pBuf, iVal) { \ |
+ (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \ |
+ assert( (pBuf)->nSpace>=(pBuf)->n ); \ |
+} |
+ |
+ |
+/* |
+** Serialize and store the "structure" record. |
+** |
+** If an error occurs, leave an error code in the Fts5Index object. If an |
+** error has already occurred, this function is a no-op. |
+*/ |
+static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ |
+ if( p->rc==SQLITE_OK ){ |
+ Fts5Buffer buf; /* Buffer to serialize record into */ |
+ int iLvl; /* Used to iterate through levels */ |
+ int iCookie; /* Cookie value to store */ |
+ |
+ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); |
+ memset(&buf, 0, sizeof(Fts5Buffer)); |
+ |
+ /* Append the current configuration cookie */ |
+ iCookie = p->pConfig->iCookie; |
+ if( iCookie<0 ) iCookie = 0; |
+ |
+ if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){ |
+ sqlite3Fts5Put32(buf.p, iCookie); |
+ buf.n = 4; |
+ fts5BufferSafeAppendVarint(&buf, pStruct->nLevel); |
+ fts5BufferSafeAppendVarint(&buf, pStruct->nSegment); |
+ fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter); |
+ } |
+ |
+ for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ |
+ int iSeg; /* Used to iterate through segments */ |
+ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; |
+ fts5BufferAppendVarint(&p->rc, &buf, pLvl->nMerge); |
+ fts5BufferAppendVarint(&p->rc, &buf, pLvl->nSeg); |
+ assert( pLvl->nMerge<=pLvl->nSeg ); |
+ |
+ for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){ |
+ fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); |
+ fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); |
+ fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); |
+ } |
+ } |
+ |
+ fts5DataWrite(p, FTS5_STRUCTURE_ROWID, buf.p, buf.n); |
+ fts5BufferFree(&buf); |
+ } |
+} |
+ |
+#if 0 |
+static void fts5DebugStructure(int*,Fts5Buffer*,Fts5Structure*); |
+static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){ |
+ int rc = SQLITE_OK; |
+ Fts5Buffer buf; |
+ memset(&buf, 0, sizeof(buf)); |
+ fts5DebugStructure(&rc, &buf, pStruct); |
+ fprintf(stdout, "%s: %s\n", zCaption, buf.p); |
+ fflush(stdout); |
+ fts5BufferFree(&buf); |
+} |
+#else |
+# define fts5PrintStructure(x,y) |
+#endif |
+ |
+static int fts5SegmentSize(Fts5StructureSegment *pSeg){ |
+ return 1 + pSeg->pgnoLast - pSeg->pgnoFirst; |
+} |
+ |
+/* |
+** Return a copy of index structure pStruct. Except, promote as many |
+** segments as possible to level iPromote. If an OOM occurs, NULL is |
+** returned. |
+*/ |
+static void fts5StructurePromoteTo( |
+ Fts5Index *p, |
+ int iPromote, |
+ int szPromote, |
+ Fts5Structure *pStruct |
+){ |
+ int il, is; |
+ Fts5StructureLevel *pOut = &pStruct->aLevel[iPromote]; |
+ |
+ if( pOut->nMerge==0 ){ |
+ for(il=iPromote+1; il<pStruct->nLevel; il++){ |
+ Fts5StructureLevel *pLvl = &pStruct->aLevel[il]; |
+ if( pLvl->nMerge ) return; |
+ for(is=pLvl->nSeg-1; is>=0; is--){ |
+ int sz = fts5SegmentSize(&pLvl->aSeg[is]); |
+ if( sz>szPromote ) return; |
+ fts5StructureExtendLevel(&p->rc, pStruct, iPromote, 1, 1); |
+ if( p->rc ) return; |
+ memcpy(pOut->aSeg, &pLvl->aSeg[is], sizeof(Fts5StructureSegment)); |
+ pOut->nSeg++; |
+ pLvl->nSeg--; |
+ } |
+ } |
+ } |
+} |
+ |
+/* |
+** A new segment has just been written to level iLvl of index structure |
+** pStruct. This function determines if any segments should be promoted |
+** as a result. Segments are promoted in two scenarios: |
+** |
+** a) If the segment just written is smaller than one or more segments |
+** within the previous populated level, it is promoted to the previous |
+** populated level. |
+** |
+** b) If the segment just written is larger than the newest segment on |
+** the next populated level, then that segment, and any other adjacent |
+** segments that are also smaller than the one just written, are |
+** promoted. |
+** |
+** If one or more segments are promoted, the structure object is updated |
+** to reflect this. |
+*/ |
+static void fts5StructurePromote( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ int iLvl, /* Index level just updated */ |
+ Fts5Structure *pStruct /* Index structure */ |
+){ |
+ if( p->rc==SQLITE_OK ){ |
+ int iTst; |
+ int iPromote = -1; |
+ int szPromote = 0; /* Promote anything this size or smaller */ |
+ Fts5StructureSegment *pSeg; /* Segment just written */ |
+ int szSeg; /* Size of segment just written */ |
+ int nSeg = pStruct->aLevel[iLvl].nSeg; |
+ |
+ if( nSeg==0 ) return; |
+ pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1]; |
+ szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst); |
+ |
+ /* Check for condition (a) */ |
+ for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--); |
+ if( iTst>=0 ){ |
+ int i; |
+ int szMax = 0; |
+ Fts5StructureLevel *pTst = &pStruct->aLevel[iTst]; |
+ assert( pTst->nMerge==0 ); |
+ for(i=0; i<pTst->nSeg; i++){ |
+ int sz = pTst->aSeg[i].pgnoLast - pTst->aSeg[i].pgnoFirst + 1; |
+ if( sz>szMax ) szMax = sz; |
+ } |
+ if( szMax>=szSeg ){ |
+ /* Condition (a) is true. Promote the newest segment on level |
+ ** iLvl to level iTst. */ |
+ iPromote = iTst; |
+ szPromote = szMax; |
+ } |
+ } |
+ |
+ /* If condition (a) is not met, assume (b) is true. StructurePromoteTo() |
+ ** is a no-op if it is not. */ |
+ if( iPromote<0 ){ |
+ iPromote = iLvl; |
+ szPromote = szSeg; |
+ } |
+ fts5StructurePromoteTo(p, iPromote, szPromote, pStruct); |
+ } |
+} |
+ |
+ |
+/* |
+** Advance the iterator passed as the only argument. If the end of the |
+** doclist-index page is reached, return non-zero. |
+*/ |
+static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ |
+ Fts5Data *pData = pLvl->pData; |
+ |
+ if( pLvl->iOff==0 ){ |
+ assert( pLvl->bEof==0 ); |
+ pLvl->iOff = 1; |
+ pLvl->iOff += fts5GetVarint32(&pData->p[1], pLvl->iLeafPgno); |
+ pLvl->iOff += fts5GetVarint(&pData->p[pLvl->iOff], (u64*)&pLvl->iRowid); |
+ pLvl->iFirstOff = pLvl->iOff; |
+ }else{ |
+ int iOff; |
+ for(iOff=pLvl->iOff; iOff<pData->nn; iOff++){ |
+ if( pData->p[iOff] ) break; |
+ } |
+ |
+ if( iOff<pData->nn ){ |
+ i64 iVal; |
+ pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; |
+ iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal); |
+ pLvl->iRowid += iVal; |
+ pLvl->iOff = iOff; |
+ }else{ |
+ pLvl->bEof = 1; |
+ } |
+ } |
+ |
+ return pLvl->bEof; |
+} |
+ |
+/* |
+** Advance the iterator passed as the only argument. |
+*/ |
+static int fts5DlidxIterNextR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){ |
+ Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl]; |
+ |
+ assert( iLvl<pIter->nLvl ); |
+ if( fts5DlidxLvlNext(pLvl) ){ |
+ if( (iLvl+1) < pIter->nLvl ){ |
+ fts5DlidxIterNextR(p, pIter, iLvl+1); |
+ if( pLvl[1].bEof==0 ){ |
+ fts5DataRelease(pLvl->pData); |
+ memset(pLvl, 0, sizeof(Fts5DlidxLvl)); |
+ pLvl->pData = fts5DataRead(p, |
+ FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno) |
+ ); |
+ if( pLvl->pData ) fts5DlidxLvlNext(pLvl); |
+ } |
+ } |
+ } |
+ |
+ return pIter->aLvl[0].bEof; |
+} |
+static int fts5DlidxIterNext(Fts5Index *p, Fts5DlidxIter *pIter){ |
+ return fts5DlidxIterNextR(p, pIter, 0); |
+} |
+ |
+/* |
+** The iterator passed as the first argument has the following fields set |
+** as follows. This function sets up the rest of the iterator so that it |
+** points to the first rowid in the doclist-index. |
+** |
+** pData: |
+** pointer to doclist-index record, |
+** |
+** When this function is called pIter->iLeafPgno is the page number the |
+** doclist is associated with (the one featuring the term). |
+*/ |
+static int fts5DlidxIterFirst(Fts5DlidxIter *pIter){ |
+ int i; |
+ for(i=0; i<pIter->nLvl; i++){ |
+ fts5DlidxLvlNext(&pIter->aLvl[i]); |
+ } |
+ return pIter->aLvl[0].bEof; |
+} |
+ |
+ |
+static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){ |
+ return p->rc!=SQLITE_OK || pIter->aLvl[0].bEof; |
+} |
+ |
+static void fts5DlidxIterLast(Fts5Index *p, Fts5DlidxIter *pIter){ |
+ int i; |
+ |
+ /* Advance each level to the last entry on the last page */ |
+ for(i=pIter->nLvl-1; p->rc==SQLITE_OK && i>=0; i--){ |
+ Fts5DlidxLvl *pLvl = &pIter->aLvl[i]; |
+ while( fts5DlidxLvlNext(pLvl)==0 ); |
+ pLvl->bEof = 0; |
+ |
+ if( i>0 ){ |
+ Fts5DlidxLvl *pChild = &pLvl[-1]; |
+ fts5DataRelease(pChild->pData); |
+ memset(pChild, 0, sizeof(Fts5DlidxLvl)); |
+ pChild->pData = fts5DataRead(p, |
+ FTS5_DLIDX_ROWID(pIter->iSegid, i-1, pLvl->iLeafPgno) |
+ ); |
+ } |
+ } |
+} |
+ |
+/* |
+** Move the iterator passed as the only argument to the previous entry. |
+*/ |
+static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){ |
+ int iOff = pLvl->iOff; |
+ |
+ assert( pLvl->bEof==0 ); |
+ if( iOff<=pLvl->iFirstOff ){ |
+ pLvl->bEof = 1; |
+ }else{ |
+ u8 *a = pLvl->pData->p; |
+ i64 iVal; |
+ int iLimit; |
+ int ii; |
+ int nZero = 0; |
+ |
+ /* Currently iOff points to the first byte of a varint. This block |
+ ** decrements iOff until it points to the first byte of the previous |
+ ** varint. Taking care not to read any memory locations that occur |
+ ** before the buffer in memory. */ |
+ iLimit = (iOff>9 ? iOff-9 : 0); |
+ for(iOff--; iOff>iLimit; iOff--){ |
+ if( (a[iOff-1] & 0x80)==0 ) break; |
+ } |
+ |
+ fts5GetVarint(&a[iOff], (u64*)&iVal); |
+ pLvl->iRowid -= iVal; |
+ pLvl->iLeafPgno--; |
+ |
+ /* Skip backwards past any 0x00 varints. */ |
+ for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){ |
+ nZero++; |
+ } |
+ if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){ |
+ /* The byte immediately before the last 0x00 byte has the 0x80 bit |
+ ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80 |
+ ** bytes before a[ii]. */ |
+ int bZero = 0; /* True if last 0x00 counts */ |
+ if( (ii-8)>=pLvl->iFirstOff ){ |
+ int j; |
+ for(j=1; j<=8 && (a[ii-j] & 0x80); j++); |
+ bZero = (j>8); |
+ } |
+ if( bZero==0 ) nZero--; |
+ } |
+ pLvl->iLeafPgno -= nZero; |
+ pLvl->iOff = iOff - nZero; |
+ } |
+ |
+ return pLvl->bEof; |
+} |
+ |
+static int fts5DlidxIterPrevR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){ |
+ Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl]; |
+ |
+ assert( iLvl<pIter->nLvl ); |
+ if( fts5DlidxLvlPrev(pLvl) ){ |
+ if( (iLvl+1) < pIter->nLvl ){ |
+ fts5DlidxIterPrevR(p, pIter, iLvl+1); |
+ if( pLvl[1].bEof==0 ){ |
+ fts5DataRelease(pLvl->pData); |
+ memset(pLvl, 0, sizeof(Fts5DlidxLvl)); |
+ pLvl->pData = fts5DataRead(p, |
+ FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno) |
+ ); |
+ if( pLvl->pData ){ |
+ while( fts5DlidxLvlNext(pLvl)==0 ); |
+ pLvl->bEof = 0; |
+ } |
+ } |
+ } |
+ } |
+ |
+ return pIter->aLvl[0].bEof; |
+} |
+static int fts5DlidxIterPrev(Fts5Index *p, Fts5DlidxIter *pIter){ |
+ return fts5DlidxIterPrevR(p, pIter, 0); |
+} |
+ |
+/* |
+** Free a doclist-index iterator object allocated by fts5DlidxIterInit(). |
+*/ |
+static void fts5DlidxIterFree(Fts5DlidxIter *pIter){ |
+ if( pIter ){ |
+ int i; |
+ for(i=0; i<pIter->nLvl; i++){ |
+ fts5DataRelease(pIter->aLvl[i].pData); |
+ } |
+ sqlite3_free(pIter); |
+ } |
+} |
+ |
+static Fts5DlidxIter *fts5DlidxIterInit( |
+ Fts5Index *p, /* Fts5 Backend to iterate within */ |
+ int bRev, /* True for ORDER BY ASC */ |
+ int iSegid, /* Segment id */ |
+ int iLeafPg /* Leaf page number to load dlidx for */ |
+){ |
+ Fts5DlidxIter *pIter = 0; |
+ int i; |
+ int bDone = 0; |
+ |
+ for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ |
+ int nByte = sizeof(Fts5DlidxIter) + i * sizeof(Fts5DlidxLvl); |
+ Fts5DlidxIter *pNew; |
+ |
+ pNew = (Fts5DlidxIter*)sqlite3_realloc(pIter, nByte); |
+ if( pNew==0 ){ |
+ p->rc = SQLITE_NOMEM; |
+ }else{ |
+ i64 iRowid = FTS5_DLIDX_ROWID(iSegid, i, iLeafPg); |
+ Fts5DlidxLvl *pLvl = &pNew->aLvl[i]; |
+ pIter = pNew; |
+ memset(pLvl, 0, sizeof(Fts5DlidxLvl)); |
+ pLvl->pData = fts5DataRead(p, iRowid); |
+ if( pLvl->pData && (pLvl->pData->p[0] & 0x0001)==0 ){ |
+ bDone = 1; |
+ } |
+ pIter->nLvl = i+1; |
+ } |
+ } |
+ |
+ if( p->rc==SQLITE_OK ){ |
+ pIter->iSegid = iSegid; |
+ if( bRev==0 ){ |
+ fts5DlidxIterFirst(pIter); |
+ }else{ |
+ fts5DlidxIterLast(p, pIter); |
+ } |
+ } |
+ |
+ if( p->rc!=SQLITE_OK ){ |
+ fts5DlidxIterFree(pIter); |
+ pIter = 0; |
+ } |
+ |
+ return pIter; |
+} |
+ |
+static i64 fts5DlidxIterRowid(Fts5DlidxIter *pIter){ |
+ return pIter->aLvl[0].iRowid; |
+} |
+static int fts5DlidxIterPgno(Fts5DlidxIter *pIter){ |
+ return pIter->aLvl[0].iLeafPgno; |
+} |
+ |
+/* |
+** Load the next leaf page into the segment iterator. |
+*/ |
+static void fts5SegIterNextPage( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5SegIter *pIter /* Iterator to advance to next page */ |
+){ |
+ Fts5Data *pLeaf; |
+ Fts5StructureSegment *pSeg = pIter->pSeg; |
+ fts5DataRelease(pIter->pLeaf); |
+ pIter->iLeafPgno++; |
+ if( pIter->pNextLeaf ){ |
+ pIter->pLeaf = pIter->pNextLeaf; |
+ pIter->pNextLeaf = 0; |
+ }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){ |
+ pIter->pLeaf = fts5LeafRead(p, |
+ FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno) |
+ ); |
+ }else{ |
+ pIter->pLeaf = 0; |
+ } |
+ pLeaf = pIter->pLeaf; |
+ |
+ if( pLeaf ){ |
+ pIter->iPgidxOff = pLeaf->szLeaf; |
+ if( fts5LeafIsTermless(pLeaf) ){ |
+ pIter->iEndofDoclist = pLeaf->nn+1; |
+ }else{ |
+ pIter->iPgidxOff += fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff], |
+ pIter->iEndofDoclist |
+ ); |
+ } |
+ } |
+} |
+ |
+/* |
+** Argument p points to a buffer containing a varint to be interpreted as a |
+** position list size field. Read the varint and return the number of bytes |
+** read. Before returning, set *pnSz to the number of bytes in the position |
+** list, and *pbDel to true if the delete flag is set, or false otherwise. |
+*/ |
+static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ |
+ int nSz; |
+ int n = 0; |
+ fts5FastGetVarint32(p, n, nSz); |
+ assert_nc( nSz>=0 ); |
+ *pnSz = nSz/2; |
+ *pbDel = nSz & 0x0001; |
+ return n; |
+} |
+ |
+/* |
+** Fts5SegIter.iLeafOffset currently points to the first byte of a |
+** position-list size field. Read the value of the field and store it |
+** in the following variables: |
+** |
+** Fts5SegIter.nPos |
+** Fts5SegIter.bDel |
+** |
+** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the |
+** position list content (if any). |
+*/ |
+static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ |
+ if( p->rc==SQLITE_OK ){ |
+ int iOff = pIter->iLeafOffset; /* Offset to read at */ |
+ ASSERT_SZLEAF_OK(pIter->pLeaf); |
+ if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ |
+ int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf); |
+ pIter->bDel = 0; |
+ pIter->nPos = 1; |
+ if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){ |
+ pIter->bDel = 1; |
+ iOff++; |
+ if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){ |
+ pIter->nPos = 1; |
+ iOff++; |
+ }else{ |
+ pIter->nPos = 0; |
+ } |
+ } |
+ }else{ |
+ int nSz; |
+ fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); |
+ pIter->bDel = (nSz & 0x0001); |
+ pIter->nPos = nSz>>1; |
+ assert_nc( pIter->nPos>=0 ); |
+ } |
+ pIter->iLeafOffset = iOff; |
+ } |
+} |
+ |
+static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ |
+ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ |
+ int iOff = pIter->iLeafOffset; |
+ |
+ ASSERT_SZLEAF_OK(pIter->pLeaf); |
+ if( iOff>=pIter->pLeaf->szLeaf ){ |
+ fts5SegIterNextPage(p, pIter); |
+ if( pIter->pLeaf==0 ){ |
+ if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; |
+ return; |
+ } |
+ iOff = 4; |
+ a = pIter->pLeaf->p; |
+ } |
+ iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); |
+ pIter->iLeafOffset = iOff; |
+} |
+ |
+/* |
+** Fts5SegIter.iLeafOffset currently points to the first byte of the |
+** "nSuffix" field of a term. Function parameter nKeep contains the value |
+** of the "nPrefix" field (if there was one - it is passed 0 if this is |
+** the first term in the segment). |
+** |
+** This function populates: |
+** |
+** Fts5SegIter.term |
+** Fts5SegIter.rowid |
+** |
+** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the content of |
+** the first position list. The position list belonging to document |
+** (Fts5SegIter.iRowid). |
+*/ |
+static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ |
+ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ |
+ int iOff = pIter->iLeafOffset; /* Offset to read at */ |
+ int nNew; /* Bytes of new data */ |
+ |
+ iOff += fts5GetVarint32(&a[iOff], nNew); |
+ if( iOff+nNew>pIter->pLeaf->nn ){ |
+ p->rc = FTS5_CORRUPT; |
+ return; |
+ } |
+ pIter->term.n = nKeep; |
+ fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); |
+ iOff += nNew; |
+ pIter->iTermLeafOffset = iOff; |
+ pIter->iTermLeafPgno = pIter->iLeafPgno; |
+ pIter->iLeafOffset = iOff; |
+ |
+ if( pIter->iPgidxOff>=pIter->pLeaf->nn ){ |
+ pIter->iEndofDoclist = pIter->pLeaf->nn+1; |
+ }else{ |
+ int nExtra; |
+ pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra); |
+ pIter->iEndofDoclist += nExtra; |
+ } |
+ |
+ fts5SegIterLoadRowid(p, pIter); |
+} |
+ |
+static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*); |
+static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*); |
+static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*); |
+ |
+static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ |
+ if( pIter->flags & FTS5_SEGITER_REVERSE ){ |
+ pIter->xNext = fts5SegIterNext_Reverse; |
+ }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ |
+ pIter->xNext = fts5SegIterNext_None; |
+ }else{ |
+ pIter->xNext = fts5SegIterNext; |
+ } |
+} |
+ |
+/* |
+** Initialize the iterator object pIter to iterate through the entries in |
+** segment pSeg. The iterator is left pointing to the first entry when |
+** this function returns. |
+** |
+** If an error occurs, Fts5Index.rc is set to an appropriate error code. If |
+** an error has already occurred when this function is called, it is a no-op. |
+*/ |
+static void fts5SegIterInit( |
+ Fts5Index *p, /* FTS index object */ |
+ Fts5StructureSegment *pSeg, /* Description of segment */ |
+ Fts5SegIter *pIter /* Object to populate */ |
+){ |
+ if( pSeg->pgnoFirst==0 ){ |
+ /* This happens if the segment is being used as an input to an incremental |
+ ** merge and all data has already been "trimmed". See function |
+ ** fts5TrimSegments() for details. In this case leave the iterator empty. |
+ ** The caller will see the (pIter->pLeaf==0) and assume the iterator is |
+ ** at EOF already. */ |
+ assert( pIter->pLeaf==0 ); |
+ return; |
+ } |
+ |
+ if( p->rc==SQLITE_OK ){ |
+ memset(pIter, 0, sizeof(*pIter)); |
+ fts5SegIterSetNext(p, pIter); |
+ pIter->pSeg = pSeg; |
+ pIter->iLeafPgno = pSeg->pgnoFirst-1; |
+ fts5SegIterNextPage(p, pIter); |
+ } |
+ |
+ if( p->rc==SQLITE_OK ){ |
+ pIter->iLeafOffset = 4; |
+ assert_nc( pIter->pLeaf->nn>4 ); |
+ assert( fts5LeafFirstTermOff(pIter->pLeaf)==4 ); |
+ pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; |
+ fts5SegIterLoadTerm(p, pIter, 0); |
+ fts5SegIterLoadNPos(p, pIter); |
+ } |
+} |
+ |
+/* |
+** This function is only ever called on iterators created by calls to |
+** Fts5IndexQuery() with the FTS5INDEX_QUERY_DESC flag set. |
+** |
+** The iterator is in an unusual state when this function is called: the |
+** Fts5SegIter.iLeafOffset variable is set to the offset of the start of |
+** the position-list size field for the first relevant rowid on the page. |
+** Fts5SegIter.rowid is set, but nPos and bDel are not. |
+** |
+** This function advances the iterator so that it points to the last |
+** relevant rowid on the page and, if necessary, initializes the |
+** aRowidOffset[] and iRowidOffset variables. At this point the iterator |
+** is in its regular state - Fts5SegIter.iLeafOffset points to the first |
+** byte of the position list content associated with said rowid. |
+*/ |
+static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ |
+ int eDetail = p->pConfig->eDetail; |
+ int n = pIter->pLeaf->szLeaf; |
+ int i = pIter->iLeafOffset; |
+ u8 *a = pIter->pLeaf->p; |
+ int iRowidOffset = 0; |
+ |
+ if( n>pIter->iEndofDoclist ){ |
+ n = pIter->iEndofDoclist; |
+ } |
+ |
+ ASSERT_SZLEAF_OK(pIter->pLeaf); |
+ while( 1 ){ |
+ i64 iDelta = 0; |
+ |
+ if( eDetail==FTS5_DETAIL_NONE ){ |
+ /* todo */ |
+ if( i<n && a[i]==0 ){ |
+ i++; |
+ if( i<n && a[i]==0 ) i++; |
+ } |
+ }else{ |
+ int nPos; |
+ int bDummy; |
+ i += fts5GetPoslistSize(&a[i], &nPos, &bDummy); |
+ i += nPos; |
+ } |
+ if( i>=n ) break; |
+ i += fts5GetVarint(&a[i], (u64*)&iDelta); |
+ pIter->iRowid += iDelta; |
+ |
+ /* If necessary, grow the pIter->aRowidOffset[] array. */ |
+ if( iRowidOffset>=pIter->nRowidOffset ){ |
+ int nNew = pIter->nRowidOffset + 8; |
+ int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int)); |
+ if( aNew==0 ){ |
+ p->rc = SQLITE_NOMEM; |
+ break; |
+ } |
+ pIter->aRowidOffset = aNew; |
+ pIter->nRowidOffset = nNew; |
+ } |
+ |
+ pIter->aRowidOffset[iRowidOffset++] = pIter->iLeafOffset; |
+ pIter->iLeafOffset = i; |
+ } |
+ pIter->iRowidOffset = iRowidOffset; |
+ fts5SegIterLoadNPos(p, pIter); |
+} |
+ |
+/* |
+** |
+*/ |
+static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ |
+ assert( pIter->flags & FTS5_SEGITER_REVERSE ); |
+ assert( pIter->flags & FTS5_SEGITER_ONETERM ); |
+ |
+ fts5DataRelease(pIter->pLeaf); |
+ pIter->pLeaf = 0; |
+ while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){ |
+ Fts5Data *pNew; |
+ pIter->iLeafPgno--; |
+ pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( |
+ pIter->pSeg->iSegid, pIter->iLeafPgno |
+ )); |
+ if( pNew ){ |
+ /* iTermLeafOffset may be equal to szLeaf if the term is the last |
+ ** thing on the page - i.e. the first rowid is on the following page. |
+ ** In this case leave pIter->pLeaf==0, this iterator is at EOF. */ |
+ if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ |
+ assert( pIter->pLeaf==0 ); |
+ if( pIter->iTermLeafOffset<pNew->szLeaf ){ |
+ pIter->pLeaf = pNew; |
+ pIter->iLeafOffset = pIter->iTermLeafOffset; |
+ } |
+ }else{ |
+ int iRowidOff; |
+ iRowidOff = fts5LeafFirstRowidOff(pNew); |
+ if( iRowidOff ){ |
+ pIter->pLeaf = pNew; |
+ pIter->iLeafOffset = iRowidOff; |
+ } |
+ } |
+ |
+ if( pIter->pLeaf ){ |
+ u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset]; |
+ pIter->iLeafOffset += fts5GetVarint(a, (u64*)&pIter->iRowid); |
+ break; |
+ }else{ |
+ fts5DataRelease(pNew); |
+ } |
+ } |
+ } |
+ |
+ if( pIter->pLeaf ){ |
+ pIter->iEndofDoclist = pIter->pLeaf->nn+1; |
+ fts5SegIterReverseInitPage(p, pIter); |
+ } |
+} |
+ |
+/* |
+** Return true if the iterator passed as the second argument currently |
+** points to a delete marker. A delete marker is an entry with a 0 byte |
+** position-list. |
+*/ |
+static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){ |
+ Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; |
+ return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); |
+} |
+ |
+/* |
+** Advance iterator pIter to the next entry. |
+** |
+** This version of fts5SegIterNext() is only used by reverse iterators. |
+*/ |
+static void fts5SegIterNext_Reverse( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5SegIter *pIter, /* Iterator to advance */ |
+ int *pbUnused /* Unused */ |
+){ |
+ assert( pIter->flags & FTS5_SEGITER_REVERSE ); |
+ assert( pIter->pNextLeaf==0 ); |
+ UNUSED_PARAM(pbUnused); |
+ |
+ if( pIter->iRowidOffset>0 ){ |
+ u8 *a = pIter->pLeaf->p; |
+ int iOff; |
+ i64 iDelta; |
+ |
+ pIter->iRowidOffset--; |
+ pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; |
+ fts5SegIterLoadNPos(p, pIter); |
+ iOff = pIter->iLeafOffset; |
+ if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ |
+ iOff += pIter->nPos; |
+ } |
+ fts5GetVarint(&a[iOff], (u64*)&iDelta); |
+ pIter->iRowid -= iDelta; |
+ }else{ |
+ fts5SegIterReverseNewPage(p, pIter); |
+ } |
+} |
+ |
+/* |
+** Advance iterator pIter to the next entry. |
+** |
+** This version of fts5SegIterNext() is only used if detail=none and the |
+** iterator is not a reverse direction iterator. |
+*/ |
+static void fts5SegIterNext_None( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5SegIter *pIter, /* Iterator to advance */ |
+ int *pbNewTerm /* OUT: Set for new term */ |
+){ |
+ int iOff; |
+ |
+ assert( p->rc==SQLITE_OK ); |
+ assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 ); |
+ assert( p->pConfig->eDetail==FTS5_DETAIL_NONE ); |
+ |
+ ASSERT_SZLEAF_OK(pIter->pLeaf); |
+ iOff = pIter->iLeafOffset; |
+ |
+ /* Next entry is on the next page */ |
+ if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ |
+ fts5SegIterNextPage(p, pIter); |
+ if( p->rc || pIter->pLeaf==0 ) return; |
+ pIter->iRowid = 0; |
+ iOff = 4; |
+ } |
+ |
+ if( iOff<pIter->iEndofDoclist ){ |
+ /* Next entry is on the current page */ |
+ i64 iDelta; |
+ iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); |
+ pIter->iLeafOffset = iOff; |
+ pIter->iRowid += iDelta; |
+ }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){ |
+ if( pIter->pSeg ){ |
+ int nKeep = 0; |
+ if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){ |
+ iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep); |
+ } |
+ pIter->iLeafOffset = iOff; |
+ fts5SegIterLoadTerm(p, pIter, nKeep); |
+ }else{ |
+ const u8 *pList = 0; |
+ const char *zTerm = 0; |
+ int nList; |
+ sqlite3Fts5HashScanNext(p->pHash); |
+ sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); |
+ if( pList==0 ) goto next_none_eof; |
+ pIter->pLeaf->p = (u8*)pList; |
+ pIter->pLeaf->nn = nList; |
+ pIter->pLeaf->szLeaf = nList; |
+ pIter->iEndofDoclist = nList; |
+ sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); |
+ pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); |
+ } |
+ |
+ if( pbNewTerm ) *pbNewTerm = 1; |
+ }else{ |
+ goto next_none_eof; |
+ } |
+ |
+ fts5SegIterLoadNPos(p, pIter); |
+ |
+ return; |
+ next_none_eof: |
+ fts5DataRelease(pIter->pLeaf); |
+ pIter->pLeaf = 0; |
+} |
+ |
+ |
+/* |
+** Advance iterator pIter to the next entry. |
+** |
+** If an error occurs, Fts5Index.rc is set to an appropriate error code. It |
+** is not considered an error if the iterator reaches EOF. If an error has |
+** already occurred when this function is called, it is a no-op. |
+*/ |
+static void fts5SegIterNext( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5SegIter *pIter, /* Iterator to advance */ |
+ int *pbNewTerm /* OUT: Set for new term */ |
+){ |
+ Fts5Data *pLeaf = pIter->pLeaf; |
+ int iOff; |
+ int bNewTerm = 0; |
+ int nKeep = 0; |
+ u8 *a; |
+ int n; |
+ |
+ assert( pbNewTerm==0 || *pbNewTerm==0 ); |
+ assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); |
+ |
+ /* Search for the end of the position list within the current page. */ |
+ a = pLeaf->p; |
+ n = pLeaf->szLeaf; |
+ |
+ ASSERT_SZLEAF_OK(pLeaf); |
+ iOff = pIter->iLeafOffset + pIter->nPos; |
+ |
+ if( iOff<n ){ |
+ /* The next entry is on the current page. */ |
+ assert_nc( iOff<=pIter->iEndofDoclist ); |
+ if( iOff>=pIter->iEndofDoclist ){ |
+ bNewTerm = 1; |
+ if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ |
+ iOff += fts5GetVarint32(&a[iOff], nKeep); |
+ } |
+ }else{ |
+ u64 iDelta; |
+ iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); |
+ pIter->iRowid += iDelta; |
+ assert_nc( iDelta>0 ); |
+ } |
+ pIter->iLeafOffset = iOff; |
+ |
+ }else if( pIter->pSeg==0 ){ |
+ const u8 *pList = 0; |
+ const char *zTerm = 0; |
+ int nList = 0; |
+ assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); |
+ if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ |
+ sqlite3Fts5HashScanNext(p->pHash); |
+ sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); |
+ } |
+ if( pList==0 ){ |
+ fts5DataRelease(pIter->pLeaf); |
+ pIter->pLeaf = 0; |
+ }else{ |
+ pIter->pLeaf->p = (u8*)pList; |
+ pIter->pLeaf->nn = nList; |
+ pIter->pLeaf->szLeaf = nList; |
+ pIter->iEndofDoclist = nList+1; |
+ sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), |
+ (u8*)zTerm); |
+ pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); |
+ *pbNewTerm = 1; |
+ } |
+ }else{ |
+ iOff = 0; |
+ /* Next entry is not on the current page */ |
+ while( iOff==0 ){ |
+ fts5SegIterNextPage(p, pIter); |
+ pLeaf = pIter->pLeaf; |
+ if( pLeaf==0 ) break; |
+ ASSERT_SZLEAF_OK(pLeaf); |
+ if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){ |
+ iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); |
+ pIter->iLeafOffset = iOff; |
+ |
+ if( pLeaf->nn>pLeaf->szLeaf ){ |
+ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( |
+ &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist |
+ ); |
+ } |
+ } |
+ else if( pLeaf->nn>pLeaf->szLeaf ){ |
+ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( |
+ &pLeaf->p[pLeaf->szLeaf], iOff |
+ ); |
+ pIter->iLeafOffset = iOff; |
+ pIter->iEndofDoclist = iOff; |
+ bNewTerm = 1; |
+ } |
+ assert_nc( iOff<pLeaf->szLeaf ); |
+ if( iOff>pLeaf->szLeaf ){ |
+ p->rc = FTS5_CORRUPT; |
+ return; |
+ } |
+ } |
+ } |
+ |
+ /* Check if the iterator is now at EOF. If so, return early. */ |
+ if( pIter->pLeaf ){ |
+ if( bNewTerm ){ |
+ if( pIter->flags & FTS5_SEGITER_ONETERM ){ |
+ fts5DataRelease(pIter->pLeaf); |
+ pIter->pLeaf = 0; |
+ }else{ |
+ fts5SegIterLoadTerm(p, pIter, nKeep); |
+ fts5SegIterLoadNPos(p, pIter); |
+ if( pbNewTerm ) *pbNewTerm = 1; |
+ } |
+ }else{ |
+ /* The following could be done by calling fts5SegIterLoadNPos(). But |
+ ** this block is particularly performance critical, so equivalent |
+ ** code is inlined. |
+ ** |
+ ** Later: Switched back to fts5SegIterLoadNPos() because it supports |
+ ** detail=none mode. Not ideal. |
+ */ |
+ int nSz; |
+ assert( p->rc==SQLITE_OK ); |
+ assert( pIter->iLeafOffset<=pIter->pLeaf->nn ); |
+ fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); |
+ pIter->bDel = (nSz & 0x0001); |
+ pIter->nPos = nSz>>1; |
+ assert_nc( pIter->nPos>=0 ); |
+ } |
+ } |
+} |
+ |
+#define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } |
+ |
+#define fts5IndexSkipVarint(a, iOff) { \ |
+ int iEnd = iOff+9; \ |
+ while( (a[iOff++] & 0x80) && iOff<iEnd ); \ |
+} |
+ |
+/* |
+** Iterator pIter currently points to the first rowid in a doclist. This |
+** function sets the iterator up so that iterates in reverse order through |
+** the doclist. |
+*/ |
+static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ |
+ Fts5DlidxIter *pDlidx = pIter->pDlidx; |
+ Fts5Data *pLast = 0; |
+ int pgnoLast = 0; |
+ |
+ if( pDlidx ){ |
+ int iSegid = pIter->pSeg->iSegid; |
+ pgnoLast = fts5DlidxIterPgno(pDlidx); |
+ pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); |
+ }else{ |
+ Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ |
+ |
+ /* Currently, Fts5SegIter.iLeafOffset points to the first byte of |
+ ** position-list content for the current rowid. Back it up so that it |
+ ** points to the start of the position-list size field. */ |
+ int iPoslist; |
+ if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ |
+ iPoslist = pIter->iTermLeafOffset; |
+ }else{ |
+ iPoslist = 4; |
+ } |
+ fts5IndexSkipVarint(pLeaf->p, iPoslist); |
+ pIter->iLeafOffset = iPoslist; |
+ |
+ /* If this condition is true then the largest rowid for the current |
+ ** term may not be stored on the current page. So search forward to |
+ ** see where said rowid really is. */ |
+ if( pIter->iEndofDoclist>=pLeaf->szLeaf ){ |
+ int pgno; |
+ Fts5StructureSegment *pSeg = pIter->pSeg; |
+ |
+ /* The last rowid in the doclist may not be on the current page. Search |
+ ** forward to find the page containing the last rowid. */ |
+ for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){ |
+ i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); |
+ Fts5Data *pNew = fts5DataRead(p, iAbs); |
+ if( pNew ){ |
+ int iRowid, bTermless; |
+ iRowid = fts5LeafFirstRowidOff(pNew); |
+ bTermless = fts5LeafIsTermless(pNew); |
+ if( iRowid ){ |
+ SWAPVAL(Fts5Data*, pNew, pLast); |
+ pgnoLast = pgno; |
+ } |
+ fts5DataRelease(pNew); |
+ if( bTermless==0 ) break; |
+ } |
+ } |
+ } |
+ } |
+ |
+ /* If pLast is NULL at this point, then the last rowid for this doclist |
+ ** lies on the page currently indicated by the iterator. In this case |
+ ** pIter->iLeafOffset is already set to point to the position-list size |
+ ** field associated with the first relevant rowid on the page. |
+ ** |
+ ** Or, if pLast is non-NULL, then it is the page that contains the last |
+ ** rowid. In this case configure the iterator so that it points to the |
+ ** first rowid on this page. |
+ */ |
+ if( pLast ){ |
+ int iOff; |
+ fts5DataRelease(pIter->pLeaf); |
+ pIter->pLeaf = pLast; |
+ pIter->iLeafPgno = pgnoLast; |
+ iOff = fts5LeafFirstRowidOff(pLast); |
+ iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); |
+ pIter->iLeafOffset = iOff; |
+ |
+ if( fts5LeafIsTermless(pLast) ){ |
+ pIter->iEndofDoclist = pLast->nn+1; |
+ }else{ |
+ pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); |
+ } |
+ |
+ } |
+ |
+ fts5SegIterReverseInitPage(p, pIter); |
+} |
+ |
+/* |
+** Iterator pIter currently points to the first rowid of a doclist. |
+** There is a doclist-index associated with the final term on the current |
+** page. If the current term is the last term on the page, load the |
+** doclist-index from disk and initialize an iterator at (pIter->pDlidx). |
+*/ |
+static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ |
+ int iSeg = pIter->pSeg->iSegid; |
+ int bRev = (pIter->flags & FTS5_SEGITER_REVERSE); |
+ Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ |
+ |
+ assert( pIter->flags & FTS5_SEGITER_ONETERM ); |
+ assert( pIter->pDlidx==0 ); |
+ |
+ /* Check if the current doclist ends on this page. If it does, return |
+ ** early without loading the doclist-index (as it belongs to a different |
+ ** term. */ |
+ if( pIter->iTermLeafPgno==pIter->iLeafPgno |
+ && pIter->iEndofDoclist<pLeaf->szLeaf |
+ ){ |
+ return; |
+ } |
+ |
+ pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); |
+} |
+ |
+/* |
+** The iterator object passed as the second argument currently contains |
+** no valid values except for the Fts5SegIter.pLeaf member variable. This |
+** function searches the leaf page for a term matching (pTerm/nTerm). |
+** |
+** If the specified term is found on the page, then the iterator is left |
+** pointing to it. If argument bGe is zero and the term is not found, |
+** the iterator is left pointing at EOF. |
+** |
+** If bGe is non-zero and the specified term is not found, then the |
+** iterator is left pointing to the smallest term in the segment that |
+** is larger than the specified term, even if this term is not on the |
+** current page. |
+*/ |
+static void fts5LeafSeek( |
+ Fts5Index *p, /* Leave any error code here */ |
+ int bGe, /* True for a >= search */ |
+ Fts5SegIter *pIter, /* Iterator to seek */ |
+ const u8 *pTerm, int nTerm /* Term to search for */ |
+){ |
+ int iOff; |
+ const u8 *a = pIter->pLeaf->p; |
+ int szLeaf = pIter->pLeaf->szLeaf; |
+ int n = pIter->pLeaf->nn; |
+ |
+ int nMatch = 0; |
+ int nKeep = 0; |
+ int nNew = 0; |
+ int iTermOff; |
+ int iPgidx; /* Current offset in pgidx */ |
+ int bEndOfPage = 0; |
+ |
+ assert( p->rc==SQLITE_OK ); |
+ |
+ iPgidx = szLeaf; |
+ iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); |
+ iOff = iTermOff; |
+ if( iOff>n ){ |
+ p->rc = FTS5_CORRUPT; |
+ return; |
+ } |
+ |
+ while( 1 ){ |
+ |
+ /* Figure out how many new bytes are in this term */ |
+ fts5FastGetVarint32(a, iOff, nNew); |
+ if( nKeep<nMatch ){ |
+ goto search_failed; |
+ } |
+ |
+ assert( nKeep>=nMatch ); |
+ if( nKeep==nMatch ){ |
+ int nCmp; |
+ int i; |
+ nCmp = MIN(nNew, nTerm-nMatch); |
+ for(i=0; i<nCmp; i++){ |
+ if( a[iOff+i]!=pTerm[nMatch+i] ) break; |
+ } |
+ nMatch += i; |
+ |
+ if( nTerm==nMatch ){ |
+ if( i==nNew ){ |
+ goto search_success; |
+ }else{ |
+ goto search_failed; |
+ } |
+ }else if( i<nNew && a[iOff+i]>pTerm[nMatch] ){ |
+ goto search_failed; |
+ } |
+ } |
+ |
+ if( iPgidx>=n ){ |
+ bEndOfPage = 1; |
+ break; |
+ } |
+ |
+ iPgidx += fts5GetVarint32(&a[iPgidx], nKeep); |
+ iTermOff += nKeep; |
+ iOff = iTermOff; |
+ |
+ if( iOff>=n ){ |
+ p->rc = FTS5_CORRUPT; |
+ return; |
+ } |
+ |
+ /* Read the nKeep field of the next term. */ |
+ fts5FastGetVarint32(a, iOff, nKeep); |
+ } |
+ |
+ search_failed: |
+ if( bGe==0 ){ |
+ fts5DataRelease(pIter->pLeaf); |
+ pIter->pLeaf = 0; |
+ return; |
+ }else if( bEndOfPage ){ |
+ do { |
+ fts5SegIterNextPage(p, pIter); |
+ if( pIter->pLeaf==0 ) return; |
+ a = pIter->pLeaf->p; |
+ if( fts5LeafIsTermless(pIter->pLeaf)==0 ){ |
+ iPgidx = pIter->pLeaf->szLeaf; |
+ iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); |
+ if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){ |
+ p->rc = FTS5_CORRUPT; |
+ }else{ |
+ nKeep = 0; |
+ iTermOff = iOff; |
+ n = pIter->pLeaf->nn; |
+ iOff += fts5GetVarint32(&a[iOff], nNew); |
+ break; |
+ } |
+ } |
+ }while( 1 ); |
+ } |
+ |
+ search_success: |
+ |
+ pIter->iLeafOffset = iOff + nNew; |
+ pIter->iTermLeafOffset = pIter->iLeafOffset; |
+ pIter->iTermLeafPgno = pIter->iLeafPgno; |
+ |
+ fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm); |
+ fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); |
+ |
+ if( iPgidx>=n ){ |
+ pIter->iEndofDoclist = pIter->pLeaf->nn+1; |
+ }else{ |
+ int nExtra; |
+ iPgidx += fts5GetVarint32(&a[iPgidx], nExtra); |
+ pIter->iEndofDoclist = iTermOff + nExtra; |
+ } |
+ pIter->iPgidxOff = iPgidx; |
+ |
+ fts5SegIterLoadRowid(p, pIter); |
+ fts5SegIterLoadNPos(p, pIter); |
+} |
+ |
+static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){ |
+ if( p->pIdxSelect==0 ){ |
+ Fts5Config *pConfig = p->pConfig; |
+ fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( |
+ "SELECT pgno FROM '%q'.'%q_idx' WHERE " |
+ "segid=? AND term<=? ORDER BY term DESC LIMIT 1", |
+ pConfig->zDb, pConfig->zName |
+ )); |
+ } |
+ return p->pIdxSelect; |
+} |
+ |
+/* |
+** Initialize the object pIter to point to term pTerm/nTerm within segment |
+** pSeg. If there is no such term in the index, the iterator is set to EOF. |
+** |
+** If an error occurs, Fts5Index.rc is set to an appropriate error code. If |
+** an error has already occurred when this function is called, it is a no-op. |
+*/ |
+static void fts5SegIterSeekInit( |
+ Fts5Index *p, /* FTS5 backend */ |
+ const u8 *pTerm, int nTerm, /* Term to seek to */ |
+ int flags, /* Mask of FTS5INDEX_XXX flags */ |
+ Fts5StructureSegment *pSeg, /* Description of segment */ |
+ Fts5SegIter *pIter /* Object to populate */ |
+){ |
+ int iPg = 1; |
+ int bGe = (flags & FTS5INDEX_QUERY_SCAN); |
+ int bDlidx = 0; /* True if there is a doclist-index */ |
+ sqlite3_stmt *pIdxSelect = 0; |
+ |
+ assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); |
+ assert( pTerm && nTerm ); |
+ memset(pIter, 0, sizeof(*pIter)); |
+ pIter->pSeg = pSeg; |
+ |
+ /* This block sets stack variable iPg to the leaf page number that may |
+ ** contain term (pTerm/nTerm), if it is present in the segment. */ |
+ pIdxSelect = fts5IdxSelectStmt(p); |
+ if( p->rc ) return; |
+ sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid); |
+ sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); |
+ if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){ |
+ i64 val = sqlite3_column_int(pIdxSelect, 0); |
+ iPg = (int)(val>>1); |
+ bDlidx = (val & 0x0001); |
+ } |
+ p->rc = sqlite3_reset(pIdxSelect); |
+ |
+ if( iPg<pSeg->pgnoFirst ){ |
+ iPg = pSeg->pgnoFirst; |
+ bDlidx = 0; |
+ } |
+ |
+ pIter->iLeafPgno = iPg - 1; |
+ fts5SegIterNextPage(p, pIter); |
+ |
+ if( pIter->pLeaf ){ |
+ fts5LeafSeek(p, bGe, pIter, pTerm, nTerm); |
+ } |
+ |
+ if( p->rc==SQLITE_OK && bGe==0 ){ |
+ pIter->flags |= FTS5_SEGITER_ONETERM; |
+ if( pIter->pLeaf ){ |
+ if( flags & FTS5INDEX_QUERY_DESC ){ |
+ pIter->flags |= FTS5_SEGITER_REVERSE; |
+ } |
+ if( bDlidx ){ |
+ fts5SegIterLoadDlidx(p, pIter); |
+ } |
+ if( flags & FTS5INDEX_QUERY_DESC ){ |
+ fts5SegIterReverse(p, pIter); |
+ } |
+ } |
+ } |
+ |
+ fts5SegIterSetNext(p, pIter); |
+ |
+ /* Either: |
+ ** |
+ ** 1) an error has occurred, or |
+ ** 2) the iterator points to EOF, or |
+ ** 3) the iterator points to an entry with term (pTerm/nTerm), or |
+ ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points |
+ ** to an entry with a term greater than or equal to (pTerm/nTerm). |
+ */ |
+ assert( p->rc!=SQLITE_OK /* 1 */ |
+ || pIter->pLeaf==0 /* 2 */ |
+ || fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)==0 /* 3 */ |
+ || (bGe && fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0) /* 4 */ |
+ ); |
+} |
+ |
+/* |
+** Initialize the object pIter to point to term pTerm/nTerm within the |
+** in-memory hash table. If there is no such term in the hash-table, the |
+** iterator is set to EOF. |
+** |
+** If an error occurs, Fts5Index.rc is set to an appropriate error code. If |
+** an error has already occurred when this function is called, it is a no-op. |
+*/ |
+static void fts5SegIterHashInit( |
+ Fts5Index *p, /* FTS5 backend */ |
+ const u8 *pTerm, int nTerm, /* Term to seek to */ |
+ int flags, /* Mask of FTS5INDEX_XXX flags */ |
+ Fts5SegIter *pIter /* Object to populate */ |
+){ |
+ const u8 *pList = 0; |
+ int nList = 0; |
+ const u8 *z = 0; |
+ int n = 0; |
+ |
+ assert( p->pHash ); |
+ assert( p->rc==SQLITE_OK ); |
+ |
+ if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){ |
+ p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); |
+ sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); |
+ n = (z ? (int)strlen((const char*)z) : 0); |
+ }else{ |
+ pIter->flags |= FTS5_SEGITER_ONETERM; |
+ sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList); |
+ z = pTerm; |
+ n = nTerm; |
+ } |
+ |
+ if( pList ){ |
+ Fts5Data *pLeaf; |
+ sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); |
+ pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); |
+ if( pLeaf==0 ) return; |
+ pLeaf->p = (u8*)pList; |
+ pLeaf->nn = pLeaf->szLeaf = nList; |
+ pIter->pLeaf = pLeaf; |
+ pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); |
+ pIter->iEndofDoclist = pLeaf->nn; |
+ |
+ if( flags & FTS5INDEX_QUERY_DESC ){ |
+ pIter->flags |= FTS5_SEGITER_REVERSE; |
+ fts5SegIterReverseInitPage(p, pIter); |
+ }else{ |
+ fts5SegIterLoadNPos(p, pIter); |
+ } |
+ } |
+ |
+ fts5SegIterSetNext(p, pIter); |
+} |
+ |
+/* |
+** Zero the iterator passed as the only argument. |
+*/ |
+static void fts5SegIterClear(Fts5SegIter *pIter){ |
+ fts5BufferFree(&pIter->term); |
+ fts5DataRelease(pIter->pLeaf); |
+ fts5DataRelease(pIter->pNextLeaf); |
+ fts5DlidxIterFree(pIter->pDlidx); |
+ sqlite3_free(pIter->aRowidOffset); |
+ memset(pIter, 0, sizeof(Fts5SegIter)); |
+} |
+ |
+#ifdef SQLITE_DEBUG |
+ |
+/* |
+** This function is used as part of the big assert() procedure implemented by |
+** fts5AssertMultiIterSetup(). It ensures that the result currently stored |
+** in *pRes is the correct result of comparing the current positions of the |
+** two iterators. |
+*/ |
+static void fts5AssertComparisonResult( |
+ Fts5Iter *pIter, |
+ Fts5SegIter *p1, |
+ Fts5SegIter *p2, |
+ Fts5CResult *pRes |
+){ |
+ int i1 = p1 - pIter->aSeg; |
+ int i2 = p2 - pIter->aSeg; |
+ |
+ if( p1->pLeaf || p2->pLeaf ){ |
+ if( p1->pLeaf==0 ){ |
+ assert( pRes->iFirst==i2 ); |
+ }else if( p2->pLeaf==0 ){ |
+ assert( pRes->iFirst==i1 ); |
+ }else{ |
+ int nMin = MIN(p1->term.n, p2->term.n); |
+ int res = memcmp(p1->term.p, p2->term.p, nMin); |
+ if( res==0 ) res = p1->term.n - p2->term.n; |
+ |
+ if( res==0 ){ |
+ assert( pRes->bTermEq==1 ); |
+ assert( p1->iRowid!=p2->iRowid ); |
+ res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : 1; |
+ }else{ |
+ assert( pRes->bTermEq==0 ); |
+ } |
+ |
+ if( res<0 ){ |
+ assert( pRes->iFirst==i1 ); |
+ }else{ |
+ assert( pRes->iFirst==i2 ); |
+ } |
+ } |
+ } |
+} |
+ |
+/* |
+** This function is a no-op unless SQLITE_DEBUG is defined when this module |
+** is compiled. In that case, this function is essentially an assert() |
+** statement used to verify that the contents of the pIter->aFirst[] array |
+** are correct. |
+*/ |
+static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){ |
+ if( p->rc==SQLITE_OK ){ |
+ Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; |
+ int i; |
+ |
+ assert( (pFirst->pLeaf==0)==pIter->base.bEof ); |
+ |
+ /* Check that pIter->iSwitchRowid is set correctly. */ |
+ for(i=0; i<pIter->nSeg; i++){ |
+ Fts5SegIter *p1 = &pIter->aSeg[i]; |
+ assert( p1==pFirst |
+ || p1->pLeaf==0 |
+ || fts5BufferCompare(&pFirst->term, &p1->term) |
+ || p1->iRowid==pIter->iSwitchRowid |
+ || (p1->iRowid<pIter->iSwitchRowid)==pIter->bRev |
+ ); |
+ } |
+ |
+ for(i=0; i<pIter->nSeg; i+=2){ |
+ Fts5SegIter *p1 = &pIter->aSeg[i]; |
+ Fts5SegIter *p2 = &pIter->aSeg[i+1]; |
+ Fts5CResult *pRes = &pIter->aFirst[(pIter->nSeg + i) / 2]; |
+ fts5AssertComparisonResult(pIter, p1, p2, pRes); |
+ } |
+ |
+ for(i=1; i<(pIter->nSeg / 2); i+=2){ |
+ Fts5SegIter *p1 = &pIter->aSeg[ pIter->aFirst[i*2].iFirst ]; |
+ Fts5SegIter *p2 = &pIter->aSeg[ pIter->aFirst[i*2+1].iFirst ]; |
+ Fts5CResult *pRes = &pIter->aFirst[i]; |
+ fts5AssertComparisonResult(pIter, p1, p2, pRes); |
+ } |
+ } |
+} |
+#else |
+# define fts5AssertMultiIterSetup(x,y) |
+#endif |
+ |
+/* |
+** Do the comparison necessary to populate pIter->aFirst[iOut]. |
+** |
+** If the returned value is non-zero, then it is the index of an entry |
+** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing |
+** to a key that is a duplicate of another, higher priority, |
+** segment-iterator in the pSeg->aSeg[] array. |
+*/ |
+static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ |
+ int i1; /* Index of left-hand Fts5SegIter */ |
+ int i2; /* Index of right-hand Fts5SegIter */ |
+ int iRes; |
+ Fts5SegIter *p1; /* Left-hand Fts5SegIter */ |
+ Fts5SegIter *p2; /* Right-hand Fts5SegIter */ |
+ Fts5CResult *pRes = &pIter->aFirst[iOut]; |
+ |
+ assert( iOut<pIter->nSeg && iOut>0 ); |
+ assert( pIter->bRev==0 || pIter->bRev==1 ); |
+ |
+ if( iOut>=(pIter->nSeg/2) ){ |
+ i1 = (iOut - pIter->nSeg/2) * 2; |
+ i2 = i1 + 1; |
+ }else{ |
+ i1 = pIter->aFirst[iOut*2].iFirst; |
+ i2 = pIter->aFirst[iOut*2+1].iFirst; |
+ } |
+ p1 = &pIter->aSeg[i1]; |
+ p2 = &pIter->aSeg[i2]; |
+ |
+ pRes->bTermEq = 0; |
+ if( p1->pLeaf==0 ){ /* If p1 is at EOF */ |
+ iRes = i2; |
+ }else if( p2->pLeaf==0 ){ /* If p2 is at EOF */ |
+ iRes = i1; |
+ }else{ |
+ int res = fts5BufferCompare(&p1->term, &p2->term); |
+ if( res==0 ){ |
+ assert( i2>i1 ); |
+ assert( i2!=0 ); |
+ pRes->bTermEq = 1; |
+ if( p1->iRowid==p2->iRowid ){ |
+ p1->bDel = p2->bDel; |
+ return i2; |
+ } |
+ res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1; |
+ } |
+ assert( res!=0 ); |
+ if( res<0 ){ |
+ iRes = i1; |
+ }else{ |
+ iRes = i2; |
+ } |
+ } |
+ |
+ pRes->iFirst = (u16)iRes; |
+ return 0; |
+} |
+ |
+/* |
+** Move the seg-iter so that it points to the first rowid on page iLeafPgno. |
+** It is an error if leaf iLeafPgno does not exist or contains no rowids. |
+*/ |
+static void fts5SegIterGotoPage( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5SegIter *pIter, /* Iterator to advance */ |
+ int iLeafPgno |
+){ |
+ assert( iLeafPgno>pIter->iLeafPgno ); |
+ |
+ if( iLeafPgno>pIter->pSeg->pgnoLast ){ |
+ p->rc = FTS5_CORRUPT; |
+ }else{ |
+ fts5DataRelease(pIter->pNextLeaf); |
+ pIter->pNextLeaf = 0; |
+ pIter->iLeafPgno = iLeafPgno-1; |
+ fts5SegIterNextPage(p, pIter); |
+ assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno ); |
+ |
+ if( p->rc==SQLITE_OK ){ |
+ int iOff; |
+ u8 *a = pIter->pLeaf->p; |
+ int n = pIter->pLeaf->szLeaf; |
+ |
+ iOff = fts5LeafFirstRowidOff(pIter->pLeaf); |
+ if( iOff<4 || iOff>=n ){ |
+ p->rc = FTS5_CORRUPT; |
+ }else{ |
+ iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); |
+ pIter->iLeafOffset = iOff; |
+ fts5SegIterLoadNPos(p, pIter); |
+ } |
+ } |
+ } |
+} |
+ |
+/* |
+** Advance the iterator passed as the second argument until it is at or |
+** past rowid iFrom. Regardless of the value of iFrom, the iterator is |
+** always advanced at least once. |
+*/ |
+static void fts5SegIterNextFrom( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5SegIter *pIter, /* Iterator to advance */ |
+ i64 iMatch /* Advance iterator at least this far */ |
+){ |
+ int bRev = (pIter->flags & FTS5_SEGITER_REVERSE); |
+ Fts5DlidxIter *pDlidx = pIter->pDlidx; |
+ int iLeafPgno = pIter->iLeafPgno; |
+ int bMove = 1; |
+ |
+ assert( pIter->flags & FTS5_SEGITER_ONETERM ); |
+ assert( pIter->pDlidx ); |
+ assert( pIter->pLeaf ); |
+ |
+ if( bRev==0 ){ |
+ while( !fts5DlidxIterEof(p, pDlidx) && iMatch>fts5DlidxIterRowid(pDlidx) ){ |
+ iLeafPgno = fts5DlidxIterPgno(pDlidx); |
+ fts5DlidxIterNext(p, pDlidx); |
+ } |
+ assert_nc( iLeafPgno>=pIter->iLeafPgno || p->rc ); |
+ if( iLeafPgno>pIter->iLeafPgno ){ |
+ fts5SegIterGotoPage(p, pIter, iLeafPgno); |
+ bMove = 0; |
+ } |
+ }else{ |
+ assert( pIter->pNextLeaf==0 ); |
+ assert( iMatch<pIter->iRowid ); |
+ while( !fts5DlidxIterEof(p, pDlidx) && iMatch<fts5DlidxIterRowid(pDlidx) ){ |
+ fts5DlidxIterPrev(p, pDlidx); |
+ } |
+ iLeafPgno = fts5DlidxIterPgno(pDlidx); |
+ |
+ assert( fts5DlidxIterEof(p, pDlidx) || iLeafPgno<=pIter->iLeafPgno ); |
+ |
+ if( iLeafPgno<pIter->iLeafPgno ){ |
+ pIter->iLeafPgno = iLeafPgno+1; |
+ fts5SegIterReverseNewPage(p, pIter); |
+ bMove = 0; |
+ } |
+ } |
+ |
+ do{ |
+ if( bMove && p->rc==SQLITE_OK ) pIter->xNext(p, pIter, 0); |
+ if( pIter->pLeaf==0 ) break; |
+ if( bRev==0 && pIter->iRowid>=iMatch ) break; |
+ if( bRev!=0 && pIter->iRowid<=iMatch ) break; |
+ bMove = 1; |
+ }while( p->rc==SQLITE_OK ); |
+} |
+ |
+ |
+/* |
+** Free the iterator object passed as the second argument. |
+*/ |
+static void fts5MultiIterFree(Fts5Iter *pIter){ |
+ if( pIter ){ |
+ int i; |
+ for(i=0; i<pIter->nSeg; i++){ |
+ fts5SegIterClear(&pIter->aSeg[i]); |
+ } |
+ fts5StructureRelease(pIter->pStruct); |
+ fts5BufferFree(&pIter->poslist); |
+ sqlite3_free(pIter); |
+ } |
+} |
+ |
+static void fts5MultiIterAdvanced( |
+ Fts5Index *p, /* FTS5 backend to iterate within */ |
+ Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ |
+ int iChanged, /* Index of sub-iterator just advanced */ |
+ int iMinset /* Minimum entry in aFirst[] to set */ |
+){ |
+ int i; |
+ for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ |
+ int iEq; |
+ if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ |
+ Fts5SegIter *pSeg = &pIter->aSeg[iEq]; |
+ assert( p->rc==SQLITE_OK ); |
+ pSeg->xNext(p, pSeg, 0); |
+ i = pIter->nSeg + iEq; |
+ } |
+ } |
+} |
+ |
+/* |
+** Sub-iterator iChanged of iterator pIter has just been advanced. It still |
+** points to the same term though - just a different rowid. This function |
+** attempts to update the contents of the pIter->aFirst[] accordingly. |
+** If it does so successfully, 0 is returned. Otherwise 1. |
+** |
+** If non-zero is returned, the caller should call fts5MultiIterAdvanced() |
+** on the iterator instead. That function does the same as this one, except |
+** that it deals with more complicated cases as well. |
+*/ |
+static int fts5MultiIterAdvanceRowid( |
+ Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ |
+ int iChanged, /* Index of sub-iterator just advanced */ |
+ Fts5SegIter **ppFirst |
+){ |
+ Fts5SegIter *pNew = &pIter->aSeg[iChanged]; |
+ |
+ if( pNew->iRowid==pIter->iSwitchRowid |
+ || (pNew->iRowid<pIter->iSwitchRowid)==pIter->bRev |
+ ){ |
+ int i; |
+ Fts5SegIter *pOther = &pIter->aSeg[iChanged ^ 0x0001]; |
+ pIter->iSwitchRowid = pIter->bRev ? SMALLEST_INT64 : LARGEST_INT64; |
+ for(i=(pIter->nSeg+iChanged)/2; 1; i=i/2){ |
+ Fts5CResult *pRes = &pIter->aFirst[i]; |
+ |
+ assert( pNew->pLeaf ); |
+ assert( pRes->bTermEq==0 || pOther->pLeaf ); |
+ |
+ if( pRes->bTermEq ){ |
+ if( pNew->iRowid==pOther->iRowid ){ |
+ return 1; |
+ }else if( (pOther->iRowid>pNew->iRowid)==pIter->bRev ){ |
+ pIter->iSwitchRowid = pOther->iRowid; |
+ pNew = pOther; |
+ }else if( (pOther->iRowid>pIter->iSwitchRowid)==pIter->bRev ){ |
+ pIter->iSwitchRowid = pOther->iRowid; |
+ } |
+ } |
+ pRes->iFirst = (u16)(pNew - pIter->aSeg); |
+ if( i==1 ) break; |
+ |
+ pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ]; |
+ } |
+ } |
+ |
+ *ppFirst = pNew; |
+ return 0; |
+} |
+ |
+/* |
+** Set the pIter->bEof variable based on the state of the sub-iterators. |
+*/ |
+static void fts5MultiIterSetEof(Fts5Iter *pIter){ |
+ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; |
+ pIter->base.bEof = pSeg->pLeaf==0; |
+ pIter->iSwitchRowid = pSeg->iRowid; |
+} |
+ |
+/* |
+** Move the iterator to the next entry. |
+** |
+** If an error occurs, an error code is left in Fts5Index.rc. It is not |
+** considered an error if the iterator reaches EOF, or if it is already at |
+** EOF when this function is called. |
+*/ |
+static void fts5MultiIterNext( |
+ Fts5Index *p, |
+ Fts5Iter *pIter, |
+ int bFrom, /* True if argument iFrom is valid */ |
+ i64 iFrom /* Advance at least as far as this */ |
+){ |
+ int bUseFrom = bFrom; |
+ assert( pIter->base.bEof==0 ); |
+ while( p->rc==SQLITE_OK ){ |
+ int iFirst = pIter->aFirst[1].iFirst; |
+ int bNewTerm = 0; |
+ Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; |
+ assert( p->rc==SQLITE_OK ); |
+ if( bUseFrom && pSeg->pDlidx ){ |
+ fts5SegIterNextFrom(p, pSeg, iFrom); |
+ }else{ |
+ pSeg->xNext(p, pSeg, &bNewTerm); |
+ } |
+ |
+ if( pSeg->pLeaf==0 || bNewTerm |
+ || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) |
+ ){ |
+ fts5MultiIterAdvanced(p, pIter, iFirst, 1); |
+ fts5MultiIterSetEof(pIter); |
+ pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; |
+ if( pSeg->pLeaf==0 ) return; |
+ } |
+ |
+ fts5AssertMultiIterSetup(p, pIter); |
+ assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); |
+ if( pIter->bSkipEmpty==0 || pSeg->nPos ){ |
+ pIter->xSetOutputs(pIter, pSeg); |
+ return; |
+ } |
+ bUseFrom = 0; |
+ } |
+} |
+ |
+static void fts5MultiIterNext2( |
+ Fts5Index *p, |
+ Fts5Iter *pIter, |
+ int *pbNewTerm /* OUT: True if *might* be new term */ |
+){ |
+ assert( pIter->bSkipEmpty ); |
+ if( p->rc==SQLITE_OK ){ |
+ do { |
+ int iFirst = pIter->aFirst[1].iFirst; |
+ Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; |
+ int bNewTerm = 0; |
+ |
+ assert( p->rc==SQLITE_OK ); |
+ pSeg->xNext(p, pSeg, &bNewTerm); |
+ if( pSeg->pLeaf==0 || bNewTerm |
+ || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) |
+ ){ |
+ fts5MultiIterAdvanced(p, pIter, iFirst, 1); |
+ fts5MultiIterSetEof(pIter); |
+ *pbNewTerm = 1; |
+ }else{ |
+ *pbNewTerm = 0; |
+ } |
+ fts5AssertMultiIterSetup(p, pIter); |
+ |
+ }while( fts5MultiIterIsEmpty(p, pIter) ); |
+ } |
+} |
+ |
+static void fts5IterSetOutputs_Noop(Fts5Iter *pUnused1, Fts5SegIter *pUnused2){ |
+ UNUSED_PARAM2(pUnused1, pUnused2); |
+} |
+ |
+static Fts5Iter *fts5MultiIterAlloc( |
+ Fts5Index *p, /* FTS5 backend to iterate within */ |
+ int nSeg |
+){ |
+ Fts5Iter *pNew; |
+ int nSlot; /* Power of two >= nSeg */ |
+ |
+ for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2); |
+ pNew = fts5IdxMalloc(p, |
+ sizeof(Fts5Iter) + /* pNew */ |
+ sizeof(Fts5SegIter) * (nSlot-1) + /* pNew->aSeg[] */ |
+ sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ |
+ ); |
+ if( pNew ){ |
+ pNew->nSeg = nSlot; |
+ pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot]; |
+ pNew->pIndex = p; |
+ pNew->xSetOutputs = fts5IterSetOutputs_Noop; |
+ } |
+ return pNew; |
+} |
+ |
+static void fts5PoslistCallback( |
+ Fts5Index *pUnused, |
+ void *pContext, |
+ const u8 *pChunk, int nChunk |
+){ |
+ UNUSED_PARAM(pUnused); |
+ assert_nc( nChunk>=0 ); |
+ if( nChunk>0 ){ |
+ fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); |
+ } |
+} |
+ |
+typedef struct PoslistCallbackCtx PoslistCallbackCtx; |
+struct PoslistCallbackCtx { |
+ Fts5Buffer *pBuf; /* Append to this buffer */ |
+ Fts5Colset *pColset; /* Restrict matches to this column */ |
+ int eState; /* See above */ |
+}; |
+ |
+typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; |
+struct PoslistOffsetsCtx { |
+ Fts5Buffer *pBuf; /* Append to this buffer */ |
+ Fts5Colset *pColset; /* Restrict matches to this column */ |
+ int iRead; |
+ int iWrite; |
+}; |
+ |
+/* |
+** TODO: Make this more efficient! |
+*/ |
+static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ |
+ int i; |
+ for(i=0; i<pColset->nCol; i++){ |
+ if( pColset->aiCol[i]==iCol ) return 1; |
+ } |
+ return 0; |
+} |
+ |
+static void fts5PoslistOffsetsCallback( |
+ Fts5Index *pUnused, |
+ void *pContext, |
+ const u8 *pChunk, int nChunk |
+){ |
+ PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; |
+ UNUSED_PARAM(pUnused); |
+ assert_nc( nChunk>=0 ); |
+ if( nChunk>0 ){ |
+ int i = 0; |
+ while( i<nChunk ){ |
+ int iVal; |
+ i += fts5GetVarint32(&pChunk[i], iVal); |
+ iVal += pCtx->iRead - 2; |
+ pCtx->iRead = iVal; |
+ if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ |
+ fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); |
+ pCtx->iWrite = iVal; |
+ } |
+ } |
+ } |
+} |
+ |
+static void fts5PoslistFilterCallback( |
+ Fts5Index *pUnused, |
+ void *pContext, |
+ const u8 *pChunk, int nChunk |
+){ |
+ PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; |
+ UNUSED_PARAM(pUnused); |
+ assert_nc( nChunk>=0 ); |
+ if( nChunk>0 ){ |
+ /* Search through to find the first varint with value 1. This is the |
+ ** start of the next columns hits. */ |
+ int i = 0; |
+ int iStart = 0; |
+ |
+ if( pCtx->eState==2 ){ |
+ int iCol; |
+ fts5FastGetVarint32(pChunk, i, iCol); |
+ if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ |
+ pCtx->eState = 1; |
+ fts5BufferSafeAppendVarint(pCtx->pBuf, 1); |
+ }else{ |
+ pCtx->eState = 0; |
+ } |
+ } |
+ |
+ do { |
+ while( i<nChunk && pChunk[i]!=0x01 ){ |
+ while( pChunk[i] & 0x80 ) i++; |
+ i++; |
+ } |
+ if( pCtx->eState ){ |
+ fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); |
+ } |
+ if( i<nChunk ){ |
+ int iCol; |
+ iStart = i; |
+ i++; |
+ if( i>=nChunk ){ |
+ pCtx->eState = 2; |
+ }else{ |
+ fts5FastGetVarint32(pChunk, i, iCol); |
+ pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol); |
+ if( pCtx->eState ){ |
+ fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); |
+ iStart = i; |
+ } |
+ } |
+ } |
+ }while( i<nChunk ); |
+ } |
+} |
+ |
+static void fts5ChunkIterate( |
+ Fts5Index *p, /* Index object */ |
+ Fts5SegIter *pSeg, /* Poslist of this iterator */ |
+ void *pCtx, /* Context pointer for xChunk callback */ |
+ void (*xChunk)(Fts5Index*, void*, const u8*, int) |
+){ |
+ int nRem = pSeg->nPos; /* Number of bytes still to come */ |
+ Fts5Data *pData = 0; |
+ u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset]; |
+ int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset); |
+ int pgno = pSeg->iLeafPgno; |
+ int pgnoSave = 0; |
+ |
+ /* This function does notmwork with detail=none databases. */ |
+ assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); |
+ |
+ if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ |
+ pgnoSave = pgno+1; |
+ } |
+ |
+ while( 1 ){ |
+ xChunk(p, pCtx, pChunk, nChunk); |
+ nRem -= nChunk; |
+ fts5DataRelease(pData); |
+ if( nRem<=0 ){ |
+ break; |
+ }else{ |
+ pgno++; |
+ pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); |
+ if( pData==0 ) break; |
+ pChunk = &pData->p[4]; |
+ nChunk = MIN(nRem, pData->szLeaf - 4); |
+ if( pgno==pgnoSave ){ |
+ assert( pSeg->pNextLeaf==0 ); |
+ pSeg->pNextLeaf = pData; |
+ pData = 0; |
+ } |
+ } |
+ } |
+} |
+ |
+/* |
+** Iterator pIter currently points to a valid entry (not EOF). This |
+** function appends the position list data for the current entry to |
+** buffer pBuf. It does not make a copy of the position-list size |
+** field. |
+*/ |
+static void fts5SegiterPoslist( |
+ Fts5Index *p, |
+ Fts5SegIter *pSeg, |
+ Fts5Colset *pColset, |
+ Fts5Buffer *pBuf |
+){ |
+ if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ |
+ if( pColset==0 ){ |
+ fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); |
+ }else{ |
+ if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ |
+ PoslistCallbackCtx sCtx; |
+ sCtx.pBuf = pBuf; |
+ sCtx.pColset = pColset; |
+ sCtx.eState = fts5IndexColsetTest(pColset, 0); |
+ assert( sCtx.eState==0 || sCtx.eState==1 ); |
+ fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); |
+ }else{ |
+ PoslistOffsetsCtx sCtx; |
+ memset(&sCtx, 0, sizeof(sCtx)); |
+ sCtx.pBuf = pBuf; |
+ sCtx.pColset = pColset; |
+ fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); |
+ } |
+ } |
+ } |
+} |
+ |
+/* |
+** IN/OUT parameter (*pa) points to a position list n bytes in size. If |
+** the position list contains entries for column iCol, then (*pa) is set |
+** to point to the sub-position-list for that column and the number of |
+** bytes in it returned. Or, if the argument position list does not |
+** contain any entries for column iCol, return 0. |
+*/ |
+static int fts5IndexExtractCol( |
+ const u8 **pa, /* IN/OUT: Pointer to poslist */ |
+ int n, /* IN: Size of poslist in bytes */ |
+ int iCol /* Column to extract from poslist */ |
+){ |
+ int iCurrent = 0; /* Anything before the first 0x01 is col 0 */ |
+ const u8 *p = *pa; |
+ const u8 *pEnd = &p[n]; /* One byte past end of position list */ |
+ |
+ while( iCol>iCurrent ){ |
+ /* Advance pointer p until it points to pEnd or an 0x01 byte that is |
+ ** not part of a varint. Note that it is not possible for a negative |
+ ** or extremely large varint to occur within an uncorrupted position |
+ ** list. So the last byte of each varint may be assumed to have a clear |
+ ** 0x80 bit. */ |
+ while( *p!=0x01 ){ |
+ while( *p++ & 0x80 ); |
+ if( p>=pEnd ) return 0; |
+ } |
+ *pa = p++; |
+ iCurrent = *p++; |
+ if( iCurrent & 0x80 ){ |
+ p--; |
+ p += fts5GetVarint32(p, iCurrent); |
+ } |
+ } |
+ if( iCol!=iCurrent ) return 0; |
+ |
+ /* Advance pointer p until it points to pEnd or an 0x01 byte that is |
+ ** not part of a varint */ |
+ while( p<pEnd && *p!=0x01 ){ |
+ while( *p++ & 0x80 ); |
+ } |
+ |
+ return p - (*pa); |
+} |
+ |
+static int fts5IndexExtractColset ( |
+ Fts5Colset *pColset, /* Colset to filter on */ |
+ const u8 *pPos, int nPos, /* Position list */ |
+ Fts5Buffer *pBuf /* Output buffer */ |
+){ |
+ int rc = SQLITE_OK; |
+ int i; |
+ |
+ fts5BufferZero(pBuf); |
+ for(i=0; i<pColset->nCol; i++){ |
+ const u8 *pSub = pPos; |
+ int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); |
+ if( nSub ){ |
+ fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** xSetOutputs callback used by detail=none tables. |
+*/ |
+static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){ |
+ assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE ); |
+ pIter->base.iRowid = pSeg->iRowid; |
+ pIter->base.nData = pSeg->nPos; |
+} |
+ |
+/* |
+** xSetOutputs callback used by detail=full and detail=col tables when no |
+** column filters are specified. |
+*/ |
+static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ |
+ pIter->base.iRowid = pSeg->iRowid; |
+ pIter->base.nData = pSeg->nPos; |
+ |
+ assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE ); |
+ assert( pIter->pColset==0 ); |
+ |
+ if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ |
+ /* All data is stored on the current page. Populate the output |
+ ** variables to point into the body of the page object. */ |
+ pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; |
+ }else{ |
+ /* The data is distributed over two or more pages. Copy it into the |
+ ** Fts5Iter.poslist buffer and then set the output pointer to point |
+ ** to this buffer. */ |
+ fts5BufferZero(&pIter->poslist); |
+ fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); |
+ pIter->base.pData = pIter->poslist.p; |
+ } |
+} |
+ |
+/* |
+** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match |
+** against no columns at all). |
+*/ |
+static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){ |
+ UNUSED_PARAM(pSeg); |
+ pIter->base.nData = 0; |
+} |
+ |
+/* |
+** xSetOutputs callback used by detail=col when there is a column filter |
+** and there are 100 or more columns. Also called as a fallback from |
+** fts5IterSetOutputs_Col100 if the column-list spans more than one page. |
+*/ |
+static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ |
+ fts5BufferZero(&pIter->poslist); |
+ fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist); |
+ pIter->base.iRowid = pSeg->iRowid; |
+ pIter->base.pData = pIter->poslist.p; |
+ pIter->base.nData = pIter->poslist.n; |
+} |
+ |
+/* |
+** xSetOutputs callback used when: |
+** |
+** * detail=col, |
+** * there is a column filter, and |
+** * the table contains 100 or fewer columns. |
+** |
+** The last point is to ensure all column numbers are stored as |
+** single-byte varints. |
+*/ |
+static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ |
+ |
+ assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); |
+ assert( pIter->pColset ); |
+ |
+ if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){ |
+ fts5IterSetOutputs_Col(pIter, pSeg); |
+ }else{ |
+ u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; |
+ u8 *pEnd = (u8*)&a[pSeg->nPos]; |
+ int iPrev = 0; |
+ int *aiCol = pIter->pColset->aiCol; |
+ int *aiColEnd = &aiCol[pIter->pColset->nCol]; |
+ |
+ u8 *aOut = pIter->poslist.p; |
+ int iPrevOut = 0; |
+ |
+ pIter->base.iRowid = pSeg->iRowid; |
+ |
+ while( a<pEnd ){ |
+ iPrev += (int)a++[0] - 2; |
+ while( *aiCol<iPrev ){ |
+ aiCol++; |
+ if( aiCol==aiColEnd ) goto setoutputs_col_out; |
+ } |
+ if( *aiCol==iPrev ){ |
+ *aOut++ = (u8)((iPrev - iPrevOut) + 2); |
+ iPrevOut = iPrev; |
+ } |
+ } |
+ |
+setoutputs_col_out: |
+ pIter->base.pData = pIter->poslist.p; |
+ pIter->base.nData = aOut - pIter->poslist.p; |
+ } |
+} |
+ |
+/* |
+** xSetOutputs callback used by detail=full when there is a column filter. |
+*/ |
+static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ |
+ Fts5Colset *pColset = pIter->pColset; |
+ pIter->base.iRowid = pSeg->iRowid; |
+ |
+ assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL ); |
+ assert( pColset ); |
+ |
+ if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ |
+ /* All data is stored on the current page. Populate the output |
+ ** variables to point into the body of the page object. */ |
+ const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; |
+ if( pColset->nCol==1 ){ |
+ pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]); |
+ pIter->base.pData = a; |
+ }else{ |
+ fts5BufferZero(&pIter->poslist); |
+ fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist); |
+ pIter->base.pData = pIter->poslist.p; |
+ pIter->base.nData = pIter->poslist.n; |
+ } |
+ }else{ |
+ /* The data is distributed over two or more pages. Copy it into the |
+ ** Fts5Iter.poslist buffer and then set the output pointer to point |
+ ** to this buffer. */ |
+ fts5BufferZero(&pIter->poslist); |
+ fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); |
+ pIter->base.pData = pIter->poslist.p; |
+ pIter->base.nData = pIter->poslist.n; |
+ } |
+} |
+ |
+static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ |
+ if( *pRc==SQLITE_OK ){ |
+ Fts5Config *pConfig = pIter->pIndex->pConfig; |
+ if( pConfig->eDetail==FTS5_DETAIL_NONE ){ |
+ pIter->xSetOutputs = fts5IterSetOutputs_None; |
+ } |
+ |
+ else if( pIter->pColset==0 ){ |
+ pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; |
+ } |
+ |
+ else if( pIter->pColset->nCol==0 ){ |
+ pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset; |
+ } |
+ |
+ else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ |
+ pIter->xSetOutputs = fts5IterSetOutputs_Full; |
+ } |
+ |
+ else{ |
+ assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); |
+ if( pConfig->nCol<=100 ){ |
+ pIter->xSetOutputs = fts5IterSetOutputs_Col100; |
+ sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol); |
+ }else{ |
+ pIter->xSetOutputs = fts5IterSetOutputs_Col; |
+ } |
+ } |
+ } |
+} |
+ |
+ |
+/* |
+** Allocate a new Fts5Iter object. |
+** |
+** The new object will be used to iterate through data in structure pStruct. |
+** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel |
+** is zero or greater, data from the first nSegment segments on level iLevel |
+** is merged. |
+** |
+** The iterator initially points to the first term/rowid entry in the |
+** iterated data. |
+*/ |
+static void fts5MultiIterNew( |
+ Fts5Index *p, /* FTS5 backend to iterate within */ |
+ Fts5Structure *pStruct, /* Structure of specific index */ |
+ int flags, /* FTS5INDEX_QUERY_XXX flags */ |
+ Fts5Colset *pColset, /* Colset to filter on (or NULL) */ |
+ const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ |
+ int iLevel, /* Level to iterate (-1 for all) */ |
+ int nSegment, /* Number of segments to merge (iLevel>=0) */ |
+ Fts5Iter **ppOut /* New object */ |
+){ |
+ int nSeg = 0; /* Number of segment-iters in use */ |
+ int iIter = 0; /* */ |
+ int iSeg; /* Used to iterate through segments */ |
+ Fts5StructureLevel *pLvl; |
+ Fts5Iter *pNew; |
+ |
+ assert( (pTerm==0 && nTerm==0) || iLevel<0 ); |
+ |
+ /* Allocate space for the new multi-seg-iterator. */ |
+ if( p->rc==SQLITE_OK ){ |
+ if( iLevel<0 ){ |
+ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); |
+ nSeg = pStruct->nSegment; |
+ nSeg += (p->pHash ? 1 : 0); |
+ }else{ |
+ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); |
+ } |
+ } |
+ *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); |
+ if( pNew==0 ) return; |
+ pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); |
+ pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY)); |
+ pNew->pStruct = pStruct; |
+ pNew->pColset = pColset; |
+ fts5StructureRef(pStruct); |
+ if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){ |
+ fts5IterSetOutputCb(&p->rc, pNew); |
+ } |
+ |
+ /* Initialize each of the component segment iterators. */ |
+ if( p->rc==SQLITE_OK ){ |
+ if( iLevel<0 ){ |
+ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; |
+ if( p->pHash ){ |
+ /* Add a segment iterator for the current contents of the hash table. */ |
+ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; |
+ fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); |
+ } |
+ for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){ |
+ for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){ |
+ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; |
+ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; |
+ if( pTerm==0 ){ |
+ fts5SegIterInit(p, pSeg, pIter); |
+ }else{ |
+ fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter); |
+ } |
+ } |
+ } |
+ }else{ |
+ pLvl = &pStruct->aLevel[iLevel]; |
+ for(iSeg=nSeg-1; iSeg>=0; iSeg--){ |
+ fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); |
+ } |
+ } |
+ assert( iIter==nSeg ); |
+ } |
+ |
+ /* If the above was successful, each component iterators now points |
+ ** to the first entry in its segment. In this case initialize the |
+ ** aFirst[] array. Or, if an error has occurred, free the iterator |
+ ** object and set the output variable to NULL. */ |
+ if( p->rc==SQLITE_OK ){ |
+ for(iIter=pNew->nSeg-1; iIter>0; iIter--){ |
+ int iEq; |
+ if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ |
+ Fts5SegIter *pSeg = &pNew->aSeg[iEq]; |
+ if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); |
+ fts5MultiIterAdvanced(p, pNew, iEq, iIter); |
+ } |
+ } |
+ fts5MultiIterSetEof(pNew); |
+ fts5AssertMultiIterSetup(p, pNew); |
+ |
+ if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ |
+ fts5MultiIterNext(p, pNew, 0, 0); |
+ }else if( pNew->base.bEof==0 ){ |
+ Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; |
+ pNew->xSetOutputs(pNew, pSeg); |
+ } |
+ |
+ }else{ |
+ fts5MultiIterFree(pNew); |
+ *ppOut = 0; |
+ } |
+} |
+ |
+/* |
+** Create an Fts5Iter that iterates through the doclist provided |
+** as the second argument. |
+*/ |
+static void fts5MultiIterNew2( |
+ Fts5Index *p, /* FTS5 backend to iterate within */ |
+ Fts5Data *pData, /* Doclist to iterate through */ |
+ int bDesc, /* True for descending rowid order */ |
+ Fts5Iter **ppOut /* New object */ |
+){ |
+ Fts5Iter *pNew; |
+ pNew = fts5MultiIterAlloc(p, 2); |
+ if( pNew ){ |
+ Fts5SegIter *pIter = &pNew->aSeg[1]; |
+ |
+ pIter->flags = FTS5_SEGITER_ONETERM; |
+ if( pData->szLeaf>0 ){ |
+ pIter->pLeaf = pData; |
+ pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); |
+ pIter->iEndofDoclist = pData->nn; |
+ pNew->aFirst[1].iFirst = 1; |
+ if( bDesc ){ |
+ pNew->bRev = 1; |
+ pIter->flags |= FTS5_SEGITER_REVERSE; |
+ fts5SegIterReverseInitPage(p, pIter); |
+ }else{ |
+ fts5SegIterLoadNPos(p, pIter); |
+ } |
+ pData = 0; |
+ }else{ |
+ pNew->base.bEof = 1; |
+ } |
+ fts5SegIterSetNext(p, pIter); |
+ |
+ *ppOut = pNew; |
+ } |
+ |
+ fts5DataRelease(pData); |
+} |
+ |
+/* |
+** Return true if the iterator is at EOF or if an error has occurred. |
+** False otherwise. |
+*/ |
+static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ |
+ assert( p->rc |
+ || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof |
+ ); |
+ return (p->rc || pIter->base.bEof); |
+} |
+ |
+/* |
+** Return the rowid of the entry that the iterator currently points |
+** to. If the iterator points to EOF when this function is called the |
+** results are undefined. |
+*/ |
+static i64 fts5MultiIterRowid(Fts5Iter *pIter){ |
+ assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); |
+ return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; |
+} |
+ |
+/* |
+** Move the iterator to the next entry at or following iMatch. |
+*/ |
+static void fts5MultiIterNextFrom( |
+ Fts5Index *p, |
+ Fts5Iter *pIter, |
+ i64 iMatch |
+){ |
+ while( 1 ){ |
+ i64 iRowid; |
+ fts5MultiIterNext(p, pIter, 1, iMatch); |
+ if( fts5MultiIterEof(p, pIter) ) break; |
+ iRowid = fts5MultiIterRowid(pIter); |
+ if( pIter->bRev==0 && iRowid>=iMatch ) break; |
+ if( pIter->bRev!=0 && iRowid<=iMatch ) break; |
+ } |
+} |
+ |
+/* |
+** Return a pointer to a buffer containing the term associated with the |
+** entry that the iterator currently points to. |
+*/ |
+static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){ |
+ Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; |
+ *pn = p->term.n; |
+ return p->term.p; |
+} |
+ |
+/* |
+** Allocate a new segment-id for the structure pStruct. The new segment |
+** id must be between 1 and 65335 inclusive, and must not be used by |
+** any currently existing segment. If a free segment id cannot be found, |
+** SQLITE_FULL is returned. |
+** |
+** If an error has already occurred, this function is a no-op. 0 is |
+** returned in this case. |
+*/ |
+static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ |
+ int iSegid = 0; |
+ |
+ if( p->rc==SQLITE_OK ){ |
+ if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ |
+ p->rc = SQLITE_FULL; |
+ }else{ |
+ /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following |
+ ** array is 63 elements, or 252 bytes, in size. */ |
+ u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32]; |
+ int iLvl, iSeg; |
+ int i; |
+ u32 mask; |
+ memset(aUsed, 0, sizeof(aUsed)); |
+ for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ |
+ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ |
+ int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; |
+ if( iId<=FTS5_MAX_SEGMENT ){ |
+ aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32); |
+ } |
+ } |
+ } |
+ |
+ for(i=0; aUsed[i]==0xFFFFFFFF; i++); |
+ mask = aUsed[i]; |
+ for(iSegid=0; mask & (1 << iSegid); iSegid++); |
+ iSegid += 1 + i*32; |
+ |
+#ifdef SQLITE_DEBUG |
+ for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ |
+ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ |
+ assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); |
+ } |
+ } |
+ assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); |
+ |
+ { |
+ sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p); |
+ if( p->rc==SQLITE_OK ){ |
+ u8 aBlob[2] = {0xff, 0xff}; |
+ sqlite3_bind_int(pIdxSelect, 1, iSegid); |
+ sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC); |
+ assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); |
+ p->rc = sqlite3_reset(pIdxSelect); |
+ } |
+ } |
+#endif |
+ } |
+ } |
+ |
+ return iSegid; |
+} |
+ |
+/* |
+** Discard all data currently cached in the hash-tables. |
+*/ |
+static void fts5IndexDiscardData(Fts5Index *p){ |
+ assert( p->pHash || p->nPendingData==0 ); |
+ if( p->pHash ){ |
+ sqlite3Fts5HashClear(p->pHash); |
+ p->nPendingData = 0; |
+ } |
+} |
+ |
+/* |
+** Return the size of the prefix, in bytes, that buffer |
+** (pNew/<length-unknown>) shares with buffer (pOld/nOld). |
+** |
+** Buffer (pNew/<length-unknown>) is guaranteed to be greater |
+** than buffer (pOld/nOld). |
+*/ |
+static int fts5PrefixCompress(int nOld, const u8 *pOld, const u8 *pNew){ |
+ int i; |
+ for(i=0; i<nOld; i++){ |
+ if( pOld[i]!=pNew[i] ) break; |
+ } |
+ return i; |
+} |
+ |
+static void fts5WriteDlidxClear( |
+ Fts5Index *p, |
+ Fts5SegWriter *pWriter, |
+ int bFlush /* If true, write dlidx to disk */ |
+){ |
+ int i; |
+ assert( bFlush==0 || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n>0) ); |
+ for(i=0; i<pWriter->nDlidx; i++){ |
+ Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i]; |
+ if( pDlidx->buf.n==0 ) break; |
+ if( bFlush ){ |
+ assert( pDlidx->pgno!=0 ); |
+ fts5DataWrite(p, |
+ FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno), |
+ pDlidx->buf.p, pDlidx->buf.n |
+ ); |
+ } |
+ sqlite3Fts5BufferZero(&pDlidx->buf); |
+ pDlidx->bPrevValid = 0; |
+ } |
+} |
+ |
+/* |
+** Grow the pWriter->aDlidx[] array to at least nLvl elements in size. |
+** Any new array elements are zeroed before returning. |
+*/ |
+static int fts5WriteDlidxGrow( |
+ Fts5Index *p, |
+ Fts5SegWriter *pWriter, |
+ int nLvl |
+){ |
+ if( p->rc==SQLITE_OK && nLvl>=pWriter->nDlidx ){ |
+ Fts5DlidxWriter *aDlidx = (Fts5DlidxWriter*)sqlite3_realloc( |
+ pWriter->aDlidx, sizeof(Fts5DlidxWriter) * nLvl |
+ ); |
+ if( aDlidx==0 ){ |
+ p->rc = SQLITE_NOMEM; |
+ }else{ |
+ int nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx); |
+ memset(&aDlidx[pWriter->nDlidx], 0, nByte); |
+ pWriter->aDlidx = aDlidx; |
+ pWriter->nDlidx = nLvl; |
+ } |
+ } |
+ return p->rc; |
+} |
+ |
+/* |
+** If the current doclist-index accumulating in pWriter->aDlidx[] is large |
+** enough, flush it to disk and return 1. Otherwise discard it and return |
+** zero. |
+*/ |
+static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){ |
+ int bFlag = 0; |
+ |
+ /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written |
+ ** to the database, also write the doclist-index to disk. */ |
+ if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ |
+ bFlag = 1; |
+ } |
+ fts5WriteDlidxClear(p, pWriter, bFlag); |
+ pWriter->nEmpty = 0; |
+ return bFlag; |
+} |
+ |
+/* |
+** This function is called whenever processing of the doclist for the |
+** last term on leaf page (pWriter->iBtPage) is completed. |
+** |
+** The doclist-index for that term is currently stored in-memory within the |
+** Fts5SegWriter.aDlidx[] array. If it is large enough, this function |
+** writes it out to disk. Or, if it is too small to bother with, discards |
+** it. |
+** |
+** Fts5SegWriter.btterm currently contains the first term on page iBtPage. |
+*/ |
+static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){ |
+ int bFlag; |
+ |
+ assert( pWriter->iBtPage || pWriter->nEmpty==0 ); |
+ if( pWriter->iBtPage==0 ) return; |
+ bFlag = fts5WriteFlushDlidx(p, pWriter); |
+ |
+ if( p->rc==SQLITE_OK ){ |
+ const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:""); |
+ /* The following was already done in fts5WriteInit(): */ |
+ /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */ |
+ sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC); |
+ sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1)); |
+ sqlite3_step(p->pIdxWriter); |
+ p->rc = sqlite3_reset(p->pIdxWriter); |
+ } |
+ pWriter->iBtPage = 0; |
+} |
+ |
+/* |
+** This is called once for each leaf page except the first that contains |
+** at least one term. Argument (nTerm/pTerm) is the split-key - a term that |
+** is larger than all terms written to earlier leaves, and equal to or |
+** smaller than the first term on the new leaf. |
+** |
+** If an error occurs, an error code is left in Fts5Index.rc. If an error |
+** has already occurred when this function is called, it is a no-op. |
+*/ |
+static void fts5WriteBtreeTerm( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5SegWriter *pWriter, /* Writer object */ |
+ int nTerm, const u8 *pTerm /* First term on new page */ |
+){ |
+ fts5WriteFlushBtree(p, pWriter); |
+ fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm); |
+ pWriter->iBtPage = pWriter->writer.pgno; |
+} |
+ |
+/* |
+** This function is called when flushing a leaf page that contains no |
+** terms at all to disk. |
+*/ |
+static void fts5WriteBtreeNoTerm( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5SegWriter *pWriter /* Writer object */ |
+){ |
+ /* If there were no rowids on the leaf page either and the doclist-index |
+ ** has already been started, append an 0x00 byte to it. */ |
+ if( pWriter->bFirstRowidInPage && pWriter->aDlidx[0].buf.n>0 ){ |
+ Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[0]; |
+ assert( pDlidx->bPrevValid ); |
+ sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, 0); |
+ } |
+ |
+ /* Increment the "number of sequential leaves without a term" counter. */ |
+ pWriter->nEmpty++; |
+} |
+ |
+static i64 fts5DlidxExtractFirstRowid(Fts5Buffer *pBuf){ |
+ i64 iRowid; |
+ int iOff; |
+ |
+ iOff = 1 + fts5GetVarint(&pBuf->p[1], (u64*)&iRowid); |
+ fts5GetVarint(&pBuf->p[iOff], (u64*)&iRowid); |
+ return iRowid; |
+} |
+ |
+/* |
+** Rowid iRowid has just been appended to the current leaf page. It is the |
+** first on the page. This function appends an appropriate entry to the current |
+** doclist-index. |
+*/ |
+static void fts5WriteDlidxAppend( |
+ Fts5Index *p, |
+ Fts5SegWriter *pWriter, |
+ i64 iRowid |
+){ |
+ int i; |
+ int bDone = 0; |
+ |
+ for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ |
+ i64 iVal; |
+ Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i]; |
+ |
+ if( pDlidx->buf.n>=p->pConfig->pgsz ){ |
+ /* The current doclist-index page is full. Write it to disk and push |
+ ** a copy of iRowid (which will become the first rowid on the next |
+ ** doclist-index leaf page) up into the next level of the b-tree |
+ ** hierarchy. If the node being flushed is currently the root node, |
+ ** also push its first rowid upwards. */ |
+ pDlidx->buf.p[0] = 0x01; /* Not the root node */ |
+ fts5DataWrite(p, |
+ FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno), |
+ pDlidx->buf.p, pDlidx->buf.n |
+ ); |
+ fts5WriteDlidxGrow(p, pWriter, i+2); |
+ pDlidx = &pWriter->aDlidx[i]; |
+ if( p->rc==SQLITE_OK && pDlidx[1].buf.n==0 ){ |
+ i64 iFirst = fts5DlidxExtractFirstRowid(&pDlidx->buf); |
+ |
+ /* This was the root node. Push its first rowid up to the new root. */ |
+ pDlidx[1].pgno = pDlidx->pgno; |
+ sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, 0); |
+ sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, pDlidx->pgno); |
+ sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, iFirst); |
+ pDlidx[1].bPrevValid = 1; |
+ pDlidx[1].iPrev = iFirst; |
+ } |
+ |
+ sqlite3Fts5BufferZero(&pDlidx->buf); |
+ pDlidx->bPrevValid = 0; |
+ pDlidx->pgno++; |
+ }else{ |
+ bDone = 1; |
+ } |
+ |
+ if( pDlidx->bPrevValid ){ |
+ iVal = iRowid - pDlidx->iPrev; |
+ }else{ |
+ i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno); |
+ assert( pDlidx->buf.n==0 ); |
+ sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone); |
+ sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno); |
+ iVal = iRowid; |
+ } |
+ |
+ sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iVal); |
+ pDlidx->bPrevValid = 1; |
+ pDlidx->iPrev = iRowid; |
+ } |
+} |
+ |
+static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ |
+ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; |
+ Fts5PageWriter *pPage = &pWriter->writer; |
+ i64 iRowid; |
+ |
+static int nCall = 0; |
+nCall++; |
+ |
+ assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); |
+ |
+ /* Set the szLeaf header field. */ |
+ assert( 0==fts5GetU16(&pPage->buf.p[2]) ); |
+ fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n); |
+ |
+ if( pWriter->bFirstTermInPage ){ |
+ /* No term was written to this page. */ |
+ assert( pPage->pgidx.n==0 ); |
+ fts5WriteBtreeNoTerm(p, pWriter); |
+ }else{ |
+ /* Append the pgidx to the page buffer. Set the szLeaf header field. */ |
+ fts5BufferAppendBlob(&p->rc, &pPage->buf, pPage->pgidx.n, pPage->pgidx.p); |
+ } |
+ |
+ /* Write the page out to disk */ |
+ iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, pPage->pgno); |
+ fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); |
+ |
+ /* Initialize the next page. */ |
+ fts5BufferZero(&pPage->buf); |
+ fts5BufferZero(&pPage->pgidx); |
+ fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); |
+ pPage->iPrevPgidx = 0; |
+ pPage->pgno++; |
+ |
+ /* Increase the leaves written counter */ |
+ pWriter->nLeafWritten++; |
+ |
+ /* The new leaf holds no terms or rowids */ |
+ pWriter->bFirstTermInPage = 1; |
+ pWriter->bFirstRowidInPage = 1; |
+} |
+ |
+/* |
+** Append term pTerm/nTerm to the segment being written by the writer passed |
+** as the second argument. |
+** |
+** If an error occurs, set the Fts5Index.rc error code. If an error has |
+** already occurred, this function is a no-op. |
+*/ |
+static void fts5WriteAppendTerm( |
+ Fts5Index *p, |
+ Fts5SegWriter *pWriter, |
+ int nTerm, const u8 *pTerm |
+){ |
+ int nPrefix; /* Bytes of prefix compression for term */ |
+ Fts5PageWriter *pPage = &pWriter->writer; |
+ Fts5Buffer *pPgidx = &pWriter->writer.pgidx; |
+ |
+ assert( p->rc==SQLITE_OK ); |
+ assert( pPage->buf.n>=4 ); |
+ assert( pPage->buf.n>4 || pWriter->bFirstTermInPage ); |
+ |
+ /* If the current leaf page is full, flush it to disk. */ |
+ if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){ |
+ if( pPage->buf.n>4 ){ |
+ fts5WriteFlushLeaf(p, pWriter); |
+ } |
+ fts5BufferGrow(&p->rc, &pPage->buf, nTerm+FTS5_DATA_PADDING); |
+ } |
+ |
+ /* TODO1: Updating pgidx here. */ |
+ pPgidx->n += sqlite3Fts5PutVarint( |
+ &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx |
+ ); |
+ pPage->iPrevPgidx = pPage->buf.n; |
+#if 0 |
+ fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n); |
+ pPgidx->n += 2; |
+#endif |
+ |
+ if( pWriter->bFirstTermInPage ){ |
+ nPrefix = 0; |
+ if( pPage->pgno!=1 ){ |
+ /* This is the first term on a leaf that is not the leftmost leaf in |
+ ** the segment b-tree. In this case it is necessary to add a term to |
+ ** the b-tree hierarchy that is (a) larger than the largest term |
+ ** already written to the segment and (b) smaller than or equal to |
+ ** this term. In other words, a prefix of (pTerm/nTerm) that is one |
+ ** byte longer than the longest prefix (pTerm/nTerm) shares with the |
+ ** previous term. |
+ ** |
+ ** Usually, the previous term is available in pPage->term. The exception |
+ ** is if this is the first term written in an incremental-merge step. |
+ ** In this case the previous term is not available, so just write a |
+ ** copy of (pTerm/nTerm) into the parent node. This is slightly |
+ ** inefficient, but still correct. */ |
+ int n = nTerm; |
+ if( pPage->term.n ){ |
+ n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm); |
+ } |
+ fts5WriteBtreeTerm(p, pWriter, n, pTerm); |
+ pPage = &pWriter->writer; |
+ } |
+ }else{ |
+ nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm); |
+ fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix); |
+ } |
+ |
+ /* Append the number of bytes of new data, then the term data itself |
+ ** to the page. */ |
+ fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix); |
+ fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix]); |
+ |
+ /* Update the Fts5PageWriter.term field. */ |
+ fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm); |
+ pWriter->bFirstTermInPage = 0; |
+ |
+ pWriter->bFirstRowidInPage = 0; |
+ pWriter->bFirstRowidInDoclist = 1; |
+ |
+ assert( p->rc || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n==0) ); |
+ pWriter->aDlidx[0].pgno = pPage->pgno; |
+} |
+ |
+/* |
+** Append a rowid and position-list size field to the writers output. |
+*/ |
+static void fts5WriteAppendRowid( |
+ Fts5Index *p, |
+ Fts5SegWriter *pWriter, |
+ i64 iRowid |
+){ |
+ if( p->rc==SQLITE_OK ){ |
+ Fts5PageWriter *pPage = &pWriter->writer; |
+ |
+ if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ |
+ fts5WriteFlushLeaf(p, pWriter); |
+ } |
+ |
+ /* If this is to be the first rowid written to the page, set the |
+ ** rowid-pointer in the page-header. Also append a value to the dlidx |
+ ** buffer, in case a doclist-index is required. */ |
+ if( pWriter->bFirstRowidInPage ){ |
+ fts5PutU16(pPage->buf.p, (u16)pPage->buf.n); |
+ fts5WriteDlidxAppend(p, pWriter, iRowid); |
+ } |
+ |
+ /* Write the rowid. */ |
+ if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){ |
+ fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid); |
+ }else{ |
+ assert( p->rc || iRowid>pWriter->iPrevRowid ); |
+ fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); |
+ } |
+ pWriter->iPrevRowid = iRowid; |
+ pWriter->bFirstRowidInDoclist = 0; |
+ pWriter->bFirstRowidInPage = 0; |
+ } |
+} |
+ |
+static void fts5WriteAppendPoslistData( |
+ Fts5Index *p, |
+ Fts5SegWriter *pWriter, |
+ const u8 *aData, |
+ int nData |
+){ |
+ Fts5PageWriter *pPage = &pWriter->writer; |
+ const u8 *a = aData; |
+ int n = nData; |
+ |
+ assert( p->pConfig->pgsz>0 ); |
+ while( p->rc==SQLITE_OK |
+ && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz |
+ ){ |
+ int nReq = p->pConfig->pgsz - pPage->buf.n - pPage->pgidx.n; |
+ int nCopy = 0; |
+ while( nCopy<nReq ){ |
+ i64 dummy; |
+ nCopy += fts5GetVarint(&a[nCopy], (u64*)&dummy); |
+ } |
+ fts5BufferAppendBlob(&p->rc, &pPage->buf, nCopy, a); |
+ a += nCopy; |
+ n -= nCopy; |
+ fts5WriteFlushLeaf(p, pWriter); |
+ } |
+ if( n>0 ){ |
+ fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a); |
+ } |
+} |
+ |
+/* |
+** Flush any data cached by the writer object to the database. Free any |
+** allocations associated with the writer. |
+*/ |
+static void fts5WriteFinish( |
+ Fts5Index *p, |
+ Fts5SegWriter *pWriter, /* Writer object */ |
+ int *pnLeaf /* OUT: Number of leaf pages in b-tree */ |
+){ |
+ int i; |
+ Fts5PageWriter *pLeaf = &pWriter->writer; |
+ if( p->rc==SQLITE_OK ){ |
+ assert( pLeaf->pgno>=1 ); |
+ if( pLeaf->buf.n>4 ){ |
+ fts5WriteFlushLeaf(p, pWriter); |
+ } |
+ *pnLeaf = pLeaf->pgno-1; |
+ if( pLeaf->pgno>1 ){ |
+ fts5WriteFlushBtree(p, pWriter); |
+ } |
+ } |
+ fts5BufferFree(&pLeaf->term); |
+ fts5BufferFree(&pLeaf->buf); |
+ fts5BufferFree(&pLeaf->pgidx); |
+ fts5BufferFree(&pWriter->btterm); |
+ |
+ for(i=0; i<pWriter->nDlidx; i++){ |
+ sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf); |
+ } |
+ sqlite3_free(pWriter->aDlidx); |
+} |
+ |
+static void fts5WriteInit( |
+ Fts5Index *p, |
+ Fts5SegWriter *pWriter, |
+ int iSegid |
+){ |
+ const int nBuffer = p->pConfig->pgsz + FTS5_DATA_PADDING; |
+ |
+ memset(pWriter, 0, sizeof(Fts5SegWriter)); |
+ pWriter->iSegid = iSegid; |
+ |
+ fts5WriteDlidxGrow(p, pWriter, 1); |
+ pWriter->writer.pgno = 1; |
+ pWriter->bFirstTermInPage = 1; |
+ pWriter->iBtPage = 1; |
+ |
+ assert( pWriter->writer.buf.n==0 ); |
+ assert( pWriter->writer.pgidx.n==0 ); |
+ |
+ /* Grow the two buffers to pgsz + padding bytes in size. */ |
+ sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.pgidx, nBuffer); |
+ sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.buf, nBuffer); |
+ |
+ if( p->pIdxWriter==0 ){ |
+ Fts5Config *pConfig = p->pConfig; |
+ fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf( |
+ "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)", |
+ pConfig->zDb, pConfig->zName |
+ )); |
+ } |
+ |
+ if( p->rc==SQLITE_OK ){ |
+ /* Initialize the 4-byte leaf-page header to 0x00. */ |
+ memset(pWriter->writer.buf.p, 0, 4); |
+ pWriter->writer.buf.n = 4; |
+ |
+ /* Bind the current output segment id to the index-writer. This is an |
+ ** optimization over binding the same value over and over as rows are |
+ ** inserted into %_idx by the current writer. */ |
+ sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); |
+ } |
+} |
+ |
+/* |
+** Iterator pIter was used to iterate through the input segments of on an |
+** incremental merge operation. This function is called if the incremental |
+** merge step has finished but the input has not been completely exhausted. |
+*/ |
+static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ |
+ int i; |
+ Fts5Buffer buf; |
+ memset(&buf, 0, sizeof(Fts5Buffer)); |
+ for(i=0; i<pIter->nSeg; i++){ |
+ Fts5SegIter *pSeg = &pIter->aSeg[i]; |
+ if( pSeg->pSeg==0 ){ |
+ /* no-op */ |
+ }else if( pSeg->pLeaf==0 ){ |
+ /* All keys from this input segment have been transfered to the output. |
+ ** Set both the first and last page-numbers to 0 to indicate that the |
+ ** segment is now empty. */ |
+ pSeg->pSeg->pgnoLast = 0; |
+ pSeg->pSeg->pgnoFirst = 0; |
+ }else{ |
+ int iOff = pSeg->iTermLeafOffset; /* Offset on new first leaf page */ |
+ i64 iLeafRowid; |
+ Fts5Data *pData; |
+ int iId = pSeg->pSeg->iSegid; |
+ u8 aHdr[4] = {0x00, 0x00, 0x00, 0x00}; |
+ |
+ iLeafRowid = FTS5_SEGMENT_ROWID(iId, pSeg->iTermLeafPgno); |
+ pData = fts5DataRead(p, iLeafRowid); |
+ if( pData ){ |
+ fts5BufferZero(&buf); |
+ fts5BufferGrow(&p->rc, &buf, pData->nn); |
+ fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); |
+ fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); |
+ fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); |
+ fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff, &pData->p[iOff]); |
+ if( p->rc==SQLITE_OK ){ |
+ /* Set the szLeaf field */ |
+ fts5PutU16(&buf.p[2], (u16)buf.n); |
+ } |
+ |
+ /* Set up the new page-index array */ |
+ fts5BufferAppendVarint(&p->rc, &buf, 4); |
+ if( pSeg->iLeafPgno==pSeg->iTermLeafPgno |
+ && pSeg->iEndofDoclist<pData->szLeaf |
+ ){ |
+ int nDiff = pData->szLeaf - pSeg->iEndofDoclist; |
+ fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4); |
+ fts5BufferAppendBlob(&p->rc, &buf, |
+ pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff] |
+ ); |
+ } |
+ |
+ fts5DataRelease(pData); |
+ pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; |
+ fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 1), iLeafRowid); |
+ fts5DataWrite(p, iLeafRowid, buf.p, buf.n); |
+ } |
+ } |
+ } |
+ fts5BufferFree(&buf); |
+} |
+ |
+static void fts5MergeChunkCallback( |
+ Fts5Index *p, |
+ void *pCtx, |
+ const u8 *pChunk, int nChunk |
+){ |
+ Fts5SegWriter *pWriter = (Fts5SegWriter*)pCtx; |
+ fts5WriteAppendPoslistData(p, pWriter, pChunk, nChunk); |
+} |
+ |
+/* |
+** |
+*/ |
+static void fts5IndexMergeLevel( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ |
+ int iLvl, /* Level to read input from */ |
+ int *pnRem /* Write up to this many output leaves */ |
+){ |
+ Fts5Structure *pStruct = *ppStruct; |
+ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; |
+ Fts5StructureLevel *pLvlOut; |
+ Fts5Iter *pIter = 0; /* Iterator to read input data */ |
+ int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ |
+ int nInput; /* Number of input segments */ |
+ Fts5SegWriter writer; /* Writer object */ |
+ Fts5StructureSegment *pSeg; /* Output segment */ |
+ Fts5Buffer term; |
+ int bOldest; /* True if the output segment is the oldest */ |
+ int eDetail = p->pConfig->eDetail; |
+ const int flags = FTS5INDEX_QUERY_NOOUTPUT; |
+ |
+ assert( iLvl<pStruct->nLevel ); |
+ assert( pLvl->nMerge<=pLvl->nSeg ); |
+ |
+ memset(&writer, 0, sizeof(Fts5SegWriter)); |
+ memset(&term, 0, sizeof(Fts5Buffer)); |
+ if( pLvl->nMerge ){ |
+ pLvlOut = &pStruct->aLevel[iLvl+1]; |
+ assert( pLvlOut->nSeg>0 ); |
+ nInput = pLvl->nMerge; |
+ pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1]; |
+ |
+ fts5WriteInit(p, &writer, pSeg->iSegid); |
+ writer.writer.pgno = pSeg->pgnoLast+1; |
+ writer.iBtPage = 0; |
+ }else{ |
+ int iSegid = fts5AllocateSegid(p, pStruct); |
+ |
+ /* Extend the Fts5Structure object as required to ensure the output |
+ ** segment exists. */ |
+ if( iLvl==pStruct->nLevel-1 ){ |
+ fts5StructureAddLevel(&p->rc, ppStruct); |
+ pStruct = *ppStruct; |
+ } |
+ fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0); |
+ if( p->rc ) return; |
+ pLvl = &pStruct->aLevel[iLvl]; |
+ pLvlOut = &pStruct->aLevel[iLvl+1]; |
+ |
+ fts5WriteInit(p, &writer, iSegid); |
+ |
+ /* Add the new segment to the output level */ |
+ pSeg = &pLvlOut->aSeg[pLvlOut->nSeg]; |
+ pLvlOut->nSeg++; |
+ pSeg->pgnoFirst = 1; |
+ pSeg->iSegid = iSegid; |
+ pStruct->nSegment++; |
+ |
+ /* Read input from all segments in the input level */ |
+ nInput = pLvl->nSeg; |
+ } |
+ bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); |
+ |
+ assert( iLvl>=0 ); |
+ for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter); |
+ fts5MultiIterEof(p, pIter)==0; |
+ fts5MultiIterNext(p, pIter, 0, 0) |
+ ){ |
+ Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; |
+ int nPos; /* position-list size field value */ |
+ int nTerm; |
+ const u8 *pTerm; |
+ |
+ /* Check for key annihilation. */ |
+ if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue; |
+ |
+ pTerm = fts5MultiIterTerm(pIter, &nTerm); |
+ if( nTerm!=term.n || memcmp(pTerm, term.p, nTerm) ){ |
+ if( pnRem && writer.nLeafWritten>nRem ){ |
+ break; |
+ } |
+ |
+ /* This is a new term. Append a term to the output segment. */ |
+ fts5WriteAppendTerm(p, &writer, nTerm, pTerm); |
+ fts5BufferSet(&p->rc, &term, nTerm, pTerm); |
+ } |
+ |
+ /* Append the rowid to the output */ |
+ /* WRITEPOSLISTSIZE */ |
+ fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter)); |
+ |
+ if( eDetail==FTS5_DETAIL_NONE ){ |
+ if( pSegIter->bDel ){ |
+ fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); |
+ if( pSegIter->nPos>0 ){ |
+ fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); |
+ } |
+ } |
+ }else{ |
+ /* Append the position-list data to the output */ |
+ nPos = pSegIter->nPos*2 + pSegIter->bDel; |
+ fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos); |
+ fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); |
+ } |
+ } |
+ |
+ /* Flush the last leaf page to disk. Set the output segment b-tree height |
+ ** and last leaf page number at the same time. */ |
+ fts5WriteFinish(p, &writer, &pSeg->pgnoLast); |
+ |
+ if( fts5MultiIterEof(p, pIter) ){ |
+ int i; |
+ |
+ /* Remove the redundant segments from the %_data table */ |
+ for(i=0; i<nInput; i++){ |
+ fts5DataRemoveSegment(p, pLvl->aSeg[i].iSegid); |
+ } |
+ |
+ /* Remove the redundant segments from the input level */ |
+ if( pLvl->nSeg!=nInput ){ |
+ int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment); |
+ memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove); |
+ } |
+ pStruct->nSegment -= nInput; |
+ pLvl->nSeg -= nInput; |
+ pLvl->nMerge = 0; |
+ if( pSeg->pgnoLast==0 ){ |
+ pLvlOut->nSeg--; |
+ pStruct->nSegment--; |
+ } |
+ }else{ |
+ assert( pSeg->pgnoLast>0 ); |
+ fts5TrimSegments(p, pIter); |
+ pLvl->nMerge = nInput; |
+ } |
+ |
+ fts5MultiIterFree(pIter); |
+ fts5BufferFree(&term); |
+ if( pnRem ) *pnRem -= writer.nLeafWritten; |
+} |
+ |
+/* |
+** Do up to nPg pages of automerge work on the index. |
+** |
+** Return true if any changes were actually made, or false otherwise. |
+*/ |
+static int fts5IndexMerge( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ |
+ int nPg, /* Pages of work to do */ |
+ int nMin /* Minimum number of segments to merge */ |
+){ |
+ int nRem = nPg; |
+ int bRet = 0; |
+ Fts5Structure *pStruct = *ppStruct; |
+ while( nRem>0 && p->rc==SQLITE_OK ){ |
+ int iLvl; /* To iterate through levels */ |
+ int iBestLvl = 0; /* Level offering the most input segments */ |
+ int nBest = 0; /* Number of input segments on best level */ |
+ |
+ /* Set iBestLvl to the level to read input segments from. */ |
+ assert( pStruct->nLevel>0 ); |
+ for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ |
+ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; |
+ if( pLvl->nMerge ){ |
+ if( pLvl->nMerge>nBest ){ |
+ iBestLvl = iLvl; |
+ nBest = pLvl->nMerge; |
+ } |
+ break; |
+ } |
+ if( pLvl->nSeg>nBest ){ |
+ nBest = pLvl->nSeg; |
+ iBestLvl = iLvl; |
+ } |
+ } |
+ |
+ /* If nBest is still 0, then the index must be empty. */ |
+#ifdef SQLITE_DEBUG |
+ for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){ |
+ assert( pStruct->aLevel[iLvl].nSeg==0 ); |
+ } |
+#endif |
+ |
+ if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){ |
+ break; |
+ } |
+ bRet = 1; |
+ fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); |
+ if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ |
+ fts5StructurePromote(p, iBestLvl+1, pStruct); |
+ } |
+ } |
+ *ppStruct = pStruct; |
+ return bRet; |
+} |
+ |
+/* |
+** A total of nLeaf leaf pages of data has just been flushed to a level-0 |
+** segment. This function updates the write-counter accordingly and, if |
+** necessary, performs incremental merge work. |
+** |
+** If an error occurs, set the Fts5Index.rc error code. If an error has |
+** already occurred, this function is a no-op. |
+*/ |
+static void fts5IndexAutomerge( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ |
+ int nLeaf /* Number of output leaves just written */ |
+){ |
+ if( p->rc==SQLITE_OK && p->pConfig->nAutomerge>0 ){ |
+ Fts5Structure *pStruct = *ppStruct; |
+ u64 nWrite; /* Initial value of write-counter */ |
+ int nWork; /* Number of work-quanta to perform */ |
+ int nRem; /* Number of leaf pages left to write */ |
+ |
+ /* Update the write-counter. While doing so, set nWork. */ |
+ nWrite = pStruct->nWriteCounter; |
+ nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit)); |
+ pStruct->nWriteCounter += nLeaf; |
+ nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel); |
+ |
+ fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge); |
+ } |
+} |
+ |
+static void fts5IndexCrisismerge( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ |
+){ |
+ const int nCrisis = p->pConfig->nCrisisMerge; |
+ Fts5Structure *pStruct = *ppStruct; |
+ int iLvl = 0; |
+ |
+ assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); |
+ while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ |
+ fts5IndexMergeLevel(p, &pStruct, iLvl, 0); |
+ assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); |
+ fts5StructurePromote(p, iLvl+1, pStruct); |
+ iLvl++; |
+ } |
+ *ppStruct = pStruct; |
+} |
+ |
+static int fts5IndexReturn(Fts5Index *p){ |
+ int rc = p->rc; |
+ p->rc = SQLITE_OK; |
+ return rc; |
+} |
+ |
+typedef struct Fts5FlushCtx Fts5FlushCtx; |
+struct Fts5FlushCtx { |
+ Fts5Index *pIdx; |
+ Fts5SegWriter writer; |
+}; |
+ |
+/* |
+** Buffer aBuf[] contains a list of varints, all small enough to fit |
+** in a 32-bit integer. Return the size of the largest prefix of this |
+** list nMax bytes or less in size. |
+*/ |
+static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ |
+ int ret; |
+ u32 dummy; |
+ ret = fts5GetVarint32(aBuf, dummy); |
+ if( ret<nMax ){ |
+ while( 1 ){ |
+ int i = fts5GetVarint32(&aBuf[ret], dummy); |
+ if( (ret + i) > nMax ) break; |
+ ret += i; |
+ } |
+ } |
+ return ret; |
+} |
+ |
+/* |
+** Flush the contents of in-memory hash table iHash to a new level-0 |
+** segment on disk. Also update the corresponding structure record. |
+** |
+** If an error occurs, set the Fts5Index.rc error code. If an error has |
+** already occurred, this function is a no-op. |
+*/ |
+static void fts5FlushOneHash(Fts5Index *p){ |
+ Fts5Hash *pHash = p->pHash; |
+ Fts5Structure *pStruct; |
+ int iSegid; |
+ int pgnoLast = 0; /* Last leaf page number in segment */ |
+ |
+ /* Obtain a reference to the index structure and allocate a new segment-id |
+ ** for the new level-0 segment. */ |
+ pStruct = fts5StructureRead(p); |
+ iSegid = fts5AllocateSegid(p, pStruct); |
+ fts5StructureInvalidate(p); |
+ |
+ if( iSegid ){ |
+ const int pgsz = p->pConfig->pgsz; |
+ int eDetail = p->pConfig->eDetail; |
+ Fts5StructureSegment *pSeg; /* New segment within pStruct */ |
+ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ |
+ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ |
+ |
+ Fts5SegWriter writer; |
+ fts5WriteInit(p, &writer, iSegid); |
+ |
+ pBuf = &writer.writer.buf; |
+ pPgidx = &writer.writer.pgidx; |
+ |
+ /* fts5WriteInit() should have initialized the buffers to (most likely) |
+ ** the maximum space required. */ |
+ assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); |
+ assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); |
+ |
+ /* Begin scanning through hash table entries. This loop runs once for each |
+ ** term/doclist currently stored within the hash table. */ |
+ if( p->rc==SQLITE_OK ){ |
+ p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); |
+ } |
+ while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ |
+ const char *zTerm; /* Buffer containing term */ |
+ const u8 *pDoclist; /* Pointer to doclist for this term */ |
+ int nDoclist; /* Size of doclist in bytes */ |
+ |
+ /* Write the term for this entry to disk. */ |
+ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); |
+ fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm); |
+ |
+ assert( writer.bFirstRowidInPage==0 ); |
+ if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ |
+ /* The entire doclist will fit on the current leaf. */ |
+ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); |
+ }else{ |
+ i64 iRowid = 0; |
+ i64 iDelta = 0; |
+ int iOff = 0; |
+ |
+ /* The entire doclist will not fit on this leaf. The following |
+ ** loop iterates through the poslists that make up the current |
+ ** doclist. */ |
+ while( p->rc==SQLITE_OK && iOff<nDoclist ){ |
+ iOff += fts5GetVarint(&pDoclist[iOff], (u64*)&iDelta); |
+ iRowid += iDelta; |
+ |
+ if( writer.bFirstRowidInPage ){ |
+ fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ |
+ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); |
+ writer.bFirstRowidInPage = 0; |
+ fts5WriteDlidxAppend(p, &writer, iRowid); |
+ }else{ |
+ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); |
+ } |
+ assert( pBuf->n<=pBuf->nSpace ); |
+ |
+ if( eDetail==FTS5_DETAIL_NONE ){ |
+ if( iOff<nDoclist && pDoclist[iOff]==0 ){ |
+ pBuf->p[pBuf->n++] = 0; |
+ iOff++; |
+ if( iOff<nDoclist && pDoclist[iOff]==0 ){ |
+ pBuf->p[pBuf->n++] = 0; |
+ iOff++; |
+ } |
+ } |
+ if( (pBuf->n + pPgidx->n)>=pgsz ){ |
+ fts5WriteFlushLeaf(p, &writer); |
+ } |
+ }else{ |
+ int bDummy; |
+ int nPos; |
+ int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); |
+ nCopy += nPos; |
+ if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ |
+ /* The entire poslist will fit on the current leaf. So copy |
+ ** it in one go. */ |
+ fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); |
+ }else{ |
+ /* The entire poslist will not fit on this leaf. So it needs |
+ ** to be broken into sections. The only qualification being |
+ ** that each varint must be stored contiguously. */ |
+ const u8 *pPoslist = &pDoclist[iOff]; |
+ int iPos = 0; |
+ while( p->rc==SQLITE_OK ){ |
+ int nSpace = pgsz - pBuf->n - pPgidx->n; |
+ int n = 0; |
+ if( (nCopy - iPos)<=nSpace ){ |
+ n = nCopy - iPos; |
+ }else{ |
+ n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); |
+ } |
+ assert( n>0 ); |
+ fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); |
+ iPos += n; |
+ if( (pBuf->n + pPgidx->n)>=pgsz ){ |
+ fts5WriteFlushLeaf(p, &writer); |
+ } |
+ if( iPos>=nCopy ) break; |
+ } |
+ } |
+ iOff += nCopy; |
+ } |
+ } |
+ } |
+ |
+ /* TODO2: Doclist terminator written here. */ |
+ /* pBuf->p[pBuf->n++] = '\0'; */ |
+ assert( pBuf->n<=pBuf->nSpace ); |
+ sqlite3Fts5HashScanNext(pHash); |
+ } |
+ sqlite3Fts5HashClear(pHash); |
+ fts5WriteFinish(p, &writer, &pgnoLast); |
+ |
+ /* Update the Fts5Structure. It is written back to the database by the |
+ ** fts5StructureRelease() call below. */ |
+ if( pStruct->nLevel==0 ){ |
+ fts5StructureAddLevel(&p->rc, &pStruct); |
+ } |
+ fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); |
+ if( p->rc==SQLITE_OK ){ |
+ pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; |
+ pSeg->iSegid = iSegid; |
+ pSeg->pgnoFirst = 1; |
+ pSeg->pgnoLast = pgnoLast; |
+ pStruct->nSegment++; |
+ } |
+ fts5StructurePromote(p, 0, pStruct); |
+ } |
+ |
+ fts5IndexAutomerge(p, &pStruct, pgnoLast); |
+ fts5IndexCrisismerge(p, &pStruct); |
+ fts5StructureWrite(p, pStruct); |
+ fts5StructureRelease(pStruct); |
+} |
+ |
+/* |
+** Flush any data stored in the in-memory hash tables to the database. |
+*/ |
+static void fts5IndexFlush(Fts5Index *p){ |
+ /* Unless it is empty, flush the hash table to disk */ |
+ if( p->nPendingData ){ |
+ assert( p->pHash ); |
+ p->nPendingData = 0; |
+ fts5FlushOneHash(p); |
+ } |
+} |
+ |
+static Fts5Structure *fts5IndexOptimizeStruct( |
+ Fts5Index *p, |
+ Fts5Structure *pStruct |
+){ |
+ Fts5Structure *pNew = 0; |
+ int nByte = sizeof(Fts5Structure); |
+ int nSeg = pStruct->nSegment; |
+ int i; |
+ |
+ /* Figure out if this structure requires optimization. A structure does |
+ ** not require optimization if either: |
+ ** |
+ ** + it consists of fewer than two segments, or |
+ ** + all segments are on the same level, or |
+ ** + all segments except one are currently inputs to a merge operation. |
+ ** |
+ ** In the first case, return NULL. In the second, increment the ref-count |
+ ** on *pStruct and return a copy of the pointer to it. |
+ */ |
+ if( nSeg<2 ) return 0; |
+ for(i=0; i<pStruct->nLevel; i++){ |
+ int nThis = pStruct->aLevel[i].nSeg; |
+ if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ |
+ fts5StructureRef(pStruct); |
+ return pStruct; |
+ } |
+ assert( pStruct->aLevel[i].nMerge<=nThis ); |
+ } |
+ |
+ nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); |
+ pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); |
+ |
+ if( pNew ){ |
+ Fts5StructureLevel *pLvl; |
+ nByte = nSeg * sizeof(Fts5StructureSegment); |
+ pNew->nLevel = pStruct->nLevel+1; |
+ pNew->nRef = 1; |
+ pNew->nWriteCounter = pStruct->nWriteCounter; |
+ pLvl = &pNew->aLevel[pStruct->nLevel]; |
+ pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); |
+ if( pLvl->aSeg ){ |
+ int iLvl, iSeg; |
+ int iSegOut = 0; |
+ /* Iterate through all segments, from oldest to newest. Add them to |
+ ** the new Fts5Level object so that pLvl->aSeg[0] is the oldest |
+ ** segment in the data structure. */ |
+ for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ |
+ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ |
+ pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg]; |
+ iSegOut++; |
+ } |
+ } |
+ pNew->nSegment = pLvl->nSeg = nSeg; |
+ }else{ |
+ sqlite3_free(pNew); |
+ pNew = 0; |
+ } |
+ } |
+ |
+ return pNew; |
+} |
+ |
+static int sqlite3Fts5IndexOptimize(Fts5Index *p){ |
+ Fts5Structure *pStruct; |
+ Fts5Structure *pNew = 0; |
+ |
+ assert( p->rc==SQLITE_OK ); |
+ fts5IndexFlush(p); |
+ pStruct = fts5StructureRead(p); |
+ fts5StructureInvalidate(p); |
+ |
+ if( pStruct ){ |
+ pNew = fts5IndexOptimizeStruct(p, pStruct); |
+ } |
+ fts5StructureRelease(pStruct); |
+ |
+ assert( pNew==0 || pNew->nSegment>0 ); |
+ if( pNew ){ |
+ int iLvl; |
+ for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){} |
+ while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ |
+ int nRem = FTS5_OPT_WORK_UNIT; |
+ fts5IndexMergeLevel(p, &pNew, iLvl, &nRem); |
+ } |
+ |
+ fts5StructureWrite(p, pNew); |
+ fts5StructureRelease(pNew); |
+ } |
+ |
+ return fts5IndexReturn(p); |
+} |
+ |
+/* |
+** This is called to implement the special "VALUES('merge', $nMerge)" |
+** INSERT command. |
+*/ |
+static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ |
+ Fts5Structure *pStruct = fts5StructureRead(p); |
+ if( pStruct ){ |
+ int nMin = p->pConfig->nUsermerge; |
+ fts5StructureInvalidate(p); |
+ if( nMerge<0 ){ |
+ Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); |
+ fts5StructureRelease(pStruct); |
+ pStruct = pNew; |
+ nMin = 2; |
+ nMerge = nMerge*-1; |
+ } |
+ if( pStruct && pStruct->nLevel ){ |
+ if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){ |
+ fts5StructureWrite(p, pStruct); |
+ } |
+ } |
+ fts5StructureRelease(pStruct); |
+ } |
+ return fts5IndexReturn(p); |
+} |
+ |
+static void fts5AppendRowid( |
+ Fts5Index *p, |
+ i64 iDelta, |
+ Fts5Iter *pUnused, |
+ Fts5Buffer *pBuf |
+){ |
+ UNUSED_PARAM(pUnused); |
+ fts5BufferAppendVarint(&p->rc, pBuf, iDelta); |
+} |
+ |
+static void fts5AppendPoslist( |
+ Fts5Index *p, |
+ i64 iDelta, |
+ Fts5Iter *pMulti, |
+ Fts5Buffer *pBuf |
+){ |
+ int nData = pMulti->base.nData; |
+ assert( nData>0 ); |
+ if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nData+9+9) ){ |
+ fts5BufferSafeAppendVarint(pBuf, iDelta); |
+ fts5BufferSafeAppendVarint(pBuf, nData*2); |
+ fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData); |
+ } |
+} |
+ |
+ |
+static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ |
+ u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; |
+ |
+ assert( pIter->aPoslist ); |
+ if( p>=pIter->aEof ){ |
+ pIter->aPoslist = 0; |
+ }else{ |
+ i64 iDelta; |
+ |
+ p += fts5GetVarint(p, (u64*)&iDelta); |
+ pIter->iRowid += iDelta; |
+ |
+ /* Read position list size */ |
+ if( p[0] & 0x80 ){ |
+ int nPos; |
+ pIter->nSize = fts5GetVarint32(p, nPos); |
+ pIter->nPoslist = (nPos>>1); |
+ }else{ |
+ pIter->nPoslist = ((int)(p[0])) >> 1; |
+ pIter->nSize = 1; |
+ } |
+ |
+ pIter->aPoslist = p; |
+ } |
+} |
+ |
+static void fts5DoclistIterInit( |
+ Fts5Buffer *pBuf, |
+ Fts5DoclistIter *pIter |
+){ |
+ memset(pIter, 0, sizeof(*pIter)); |
+ pIter->aPoslist = pBuf->p; |
+ pIter->aEof = &pBuf->p[pBuf->n]; |
+ fts5DoclistIterNext(pIter); |
+} |
+ |
+#if 0 |
+/* |
+** Append a doclist to buffer pBuf. |
+** |
+** This function assumes that space within the buffer has already been |
+** allocated. |
+*/ |
+static void fts5MergeAppendDocid( |
+ Fts5Buffer *pBuf, /* Buffer to write to */ |
+ i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */ |
+ i64 iRowid /* Rowid to append */ |
+){ |
+ assert( pBuf->n!=0 || (*piLastRowid)==0 ); |
+ fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid); |
+ *piLastRowid = iRowid; |
+} |
+#endif |
+ |
+#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ |
+ assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ |
+ fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ |
+ (iLastRowid) = (iRowid); \ |
+} |
+ |
+/* |
+** Swap the contents of buffer *p1 with that of *p2. |
+*/ |
+static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ |
+ Fts5Buffer tmp = *p1; |
+ *p1 = *p2; |
+ *p2 = tmp; |
+} |
+ |
+static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){ |
+ int i = *piOff; |
+ if( i>=pBuf->n ){ |
+ *piOff = -1; |
+ }else{ |
+ u64 iVal; |
+ *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal); |
+ *piRowid += iVal; |
+ } |
+} |
+ |
+/* |
+** This is the equivalent of fts5MergePrefixLists() for detail=none mode. |
+** In this case the buffers consist of a delta-encoded list of rowids only. |
+*/ |
+static void fts5MergeRowidLists( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5Buffer *p1, /* First list to merge */ |
+ Fts5Buffer *p2 /* Second list to merge */ |
+){ |
+ int i1 = 0; |
+ int i2 = 0; |
+ i64 iRowid1 = 0; |
+ i64 iRowid2 = 0; |
+ i64 iOut = 0; |
+ |
+ Fts5Buffer out; |
+ memset(&out, 0, sizeof(out)); |
+ sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); |
+ if( p->rc ) return; |
+ |
+ fts5NextRowid(p1, &i1, &iRowid1); |
+ fts5NextRowid(p2, &i2, &iRowid2); |
+ while( i1>=0 || i2>=0 ){ |
+ if( i1>=0 && (i2<0 || iRowid1<iRowid2) ){ |
+ assert( iOut==0 || iRowid1>iOut ); |
+ fts5BufferSafeAppendVarint(&out, iRowid1 - iOut); |
+ iOut = iRowid1; |
+ fts5NextRowid(p1, &i1, &iRowid1); |
+ }else{ |
+ assert( iOut==0 || iRowid2>iOut ); |
+ fts5BufferSafeAppendVarint(&out, iRowid2 - iOut); |
+ iOut = iRowid2; |
+ if( i1>=0 && iRowid1==iRowid2 ){ |
+ fts5NextRowid(p1, &i1, &iRowid1); |
+ } |
+ fts5NextRowid(p2, &i2, &iRowid2); |
+ } |
+ } |
+ |
+ fts5BufferSwap(&out, p1); |
+ fts5BufferFree(&out); |
+} |
+ |
+/* |
+** Buffers p1 and p2 contain doclists. This function merges the content |
+** of the two doclists together and sets buffer p1 to the result before |
+** returning. |
+** |
+** If an error occurs, an error code is left in p->rc. If an error has |
+** already occurred, this function is a no-op. |
+*/ |
+static void fts5MergePrefixLists( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5Buffer *p1, /* First list to merge */ |
+ Fts5Buffer *p2 /* Second list to merge */ |
+){ |
+ if( p2->n ){ |
+ i64 iLastRowid = 0; |
+ Fts5DoclistIter i1; |
+ Fts5DoclistIter i2; |
+ Fts5Buffer out = {0, 0, 0}; |
+ Fts5Buffer tmp = {0, 0, 0}; |
+ |
+ if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return; |
+ fts5DoclistIterInit(p1, &i1); |
+ fts5DoclistIterInit(p2, &i2); |
+ |
+ while( 1 ){ |
+ if( i1.iRowid<i2.iRowid ){ |
+ /* Copy entry from i1 */ |
+ fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); |
+ fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize); |
+ fts5DoclistIterNext(&i1); |
+ if( i1.aPoslist==0 ) break; |
+ } |
+ else if( i2.iRowid!=i1.iRowid ){ |
+ /* Copy entry from i2 */ |
+ fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); |
+ fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize); |
+ fts5DoclistIterNext(&i2); |
+ if( i2.aPoslist==0 ) break; |
+ } |
+ else{ |
+ /* Merge the two position lists. */ |
+ i64 iPos1 = 0; |
+ i64 iPos2 = 0; |
+ int iOff1 = 0; |
+ int iOff2 = 0; |
+ u8 *a1 = &i1.aPoslist[i1.nSize]; |
+ u8 *a2 = &i2.aPoslist[i2.nSize]; |
+ |
+ i64 iPrev = 0; |
+ Fts5PoslistWriter writer; |
+ memset(&writer, 0, sizeof(writer)); |
+ |
+ fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); |
+ fts5BufferZero(&tmp); |
+ sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist); |
+ if( p->rc ) break; |
+ |
+ sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); |
+ sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); |
+ assert( iPos1>=0 && iPos2>=0 ); |
+ |
+ if( iPos1<iPos2 ){ |
+ sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1); |
+ sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); |
+ }else{ |
+ sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); |
+ sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); |
+ } |
+ |
+ if( iPos1>=0 && iPos2>=0 ){ |
+ while( 1 ){ |
+ if( iPos1<iPos2 ){ |
+ if( iPos1!=iPrev ){ |
+ sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1); |
+ } |
+ sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); |
+ if( iPos1<0 ) break; |
+ }else{ |
+ assert( iPos2!=iPrev ); |
+ sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); |
+ sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); |
+ if( iPos2<0 ) break; |
+ } |
+ } |
+ } |
+ |
+ if( iPos1>=0 ){ |
+ if( iPos1!=iPrev ){ |
+ sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1); |
+ } |
+ fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1); |
+ }else{ |
+ assert( iPos2>=0 && iPos2!=iPrev ); |
+ sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); |
+ fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2); |
+ } |
+ |
+ /* WRITEPOSLISTSIZE */ |
+ fts5BufferSafeAppendVarint(&out, tmp.n * 2); |
+ fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); |
+ fts5DoclistIterNext(&i1); |
+ fts5DoclistIterNext(&i2); |
+ if( i1.aPoslist==0 || i2.aPoslist==0 ) break; |
+ } |
+ } |
+ |
+ if( i1.aPoslist ){ |
+ fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); |
+ fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist); |
+ } |
+ else if( i2.aPoslist ){ |
+ fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); |
+ fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist); |
+ } |
+ |
+ fts5BufferSet(&p->rc, p1, out.n, out.p); |
+ fts5BufferFree(&tmp); |
+ fts5BufferFree(&out); |
+ } |
+} |
+ |
+static void fts5SetupPrefixIter( |
+ Fts5Index *p, /* Index to read from */ |
+ int bDesc, /* True for "ORDER BY rowid DESC" */ |
+ const u8 *pToken, /* Buffer containing prefix to match */ |
+ int nToken, /* Size of buffer pToken in bytes */ |
+ Fts5Colset *pColset, /* Restrict matches to these columns */ |
+ Fts5Iter **ppIter /* OUT: New iterator */ |
+){ |
+ Fts5Structure *pStruct; |
+ Fts5Buffer *aBuf; |
+ const int nBuf = 32; |
+ |
+ void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); |
+ void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); |
+ if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ |
+ xMerge = fts5MergeRowidLists; |
+ xAppend = fts5AppendRowid; |
+ }else{ |
+ xMerge = fts5MergePrefixLists; |
+ xAppend = fts5AppendPoslist; |
+ } |
+ |
+ aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); |
+ pStruct = fts5StructureRead(p); |
+ |
+ if( aBuf && pStruct ){ |
+ const int flags = FTS5INDEX_QUERY_SCAN |
+ | FTS5INDEX_QUERY_SKIPEMPTY |
+ | FTS5INDEX_QUERY_NOOUTPUT; |
+ int i; |
+ i64 iLastRowid = 0; |
+ Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ |
+ Fts5Data *pData; |
+ Fts5Buffer doclist; |
+ int bNewTerm = 1; |
+ |
+ memset(&doclist, 0, sizeof(doclist)); |
+ fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); |
+ fts5IterSetOutputCb(&p->rc, p1); |
+ for( /* no-op */ ; |
+ fts5MultiIterEof(p, p1)==0; |
+ fts5MultiIterNext2(p, p1, &bNewTerm) |
+ ){ |
+ Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; |
+ int nTerm = pSeg->term.n; |
+ const u8 *pTerm = pSeg->term.p; |
+ p1->xSetOutputs(p1, pSeg); |
+ |
+ assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); |
+ if( bNewTerm ){ |
+ if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break; |
+ } |
+ |
+ if( p1->base.nData==0 ) continue; |
+ |
+ if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ |
+ for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ |
+ assert( i<nBuf ); |
+ if( aBuf[i].n==0 ){ |
+ fts5BufferSwap(&doclist, &aBuf[i]); |
+ fts5BufferZero(&doclist); |
+ }else{ |
+ xMerge(p, &doclist, &aBuf[i]); |
+ fts5BufferZero(&aBuf[i]); |
+ } |
+ } |
+ iLastRowid = 0; |
+ } |
+ |
+ xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); |
+ iLastRowid = p1->base.iRowid; |
+ } |
+ |
+ for(i=0; i<nBuf; i++){ |
+ if( p->rc==SQLITE_OK ){ |
+ xMerge(p, &doclist, &aBuf[i]); |
+ } |
+ fts5BufferFree(&aBuf[i]); |
+ } |
+ fts5MultiIterFree(p1); |
+ |
+ pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n); |
+ if( pData ){ |
+ pData->p = (u8*)&pData[1]; |
+ pData->nn = pData->szLeaf = doclist.n; |
+ memcpy(pData->p, doclist.p, doclist.n); |
+ fts5MultiIterNew2(p, pData, bDesc, ppIter); |
+ } |
+ fts5BufferFree(&doclist); |
+ } |
+ |
+ fts5StructureRelease(pStruct); |
+ sqlite3_free(aBuf); |
+} |
+ |
+ |
+/* |
+** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain |
+** to the document with rowid iRowid. |
+*/ |
+static int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ |
+ assert( p->rc==SQLITE_OK ); |
+ |
+ /* Allocate the hash table if it has not already been allocated */ |
+ if( p->pHash==0 ){ |
+ p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData); |
+ } |
+ |
+ /* Flush the hash table to disk if required */ |
+ if( iRowid<p->iWriteRowid |
+ || (iRowid==p->iWriteRowid && p->bDelete==0) |
+ || (p->nPendingData > p->pConfig->nHashSize) |
+ ){ |
+ fts5IndexFlush(p); |
+ } |
+ |
+ p->iWriteRowid = iRowid; |
+ p->bDelete = bDelete; |
+ return fts5IndexReturn(p); |
+} |
+ |
+/* |
+** Commit data to disk. |
+*/ |
+static int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){ |
+ assert( p->rc==SQLITE_OK ); |
+ fts5IndexFlush(p); |
+ if( bCommit ) fts5CloseReader(p); |
+ return fts5IndexReturn(p); |
+} |
+ |
+/* |
+** Discard any data stored in the in-memory hash tables. Do not write it |
+** to the database. Additionally, assume that the contents of the %_data |
+** table may have changed on disk. So any in-memory caches of %_data |
+** records must be invalidated. |
+*/ |
+static int sqlite3Fts5IndexRollback(Fts5Index *p){ |
+ fts5CloseReader(p); |
+ fts5IndexDiscardData(p); |
+ fts5StructureInvalidate(p); |
+ /* assert( p->rc==SQLITE_OK ); */ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** The %_data table is completely empty when this function is called. This |
+** function populates it with the initial structure objects for each index, |
+** and the initial version of the "averages" record (a zero-byte blob). |
+*/ |
+static int sqlite3Fts5IndexReinit(Fts5Index *p){ |
+ Fts5Structure s; |
+ fts5StructureInvalidate(p); |
+ memset(&s, 0, sizeof(Fts5Structure)); |
+ fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); |
+ fts5StructureWrite(p, &s); |
+ return fts5IndexReturn(p); |
+} |
+ |
+/* |
+** Open a new Fts5Index handle. If the bCreate argument is true, create |
+** and initialize the underlying %_data table. |
+** |
+** If successful, set *pp to point to the new object and return SQLITE_OK. |
+** Otherwise, set *pp to NULL and return an SQLite error code. |
+*/ |
+static int sqlite3Fts5IndexOpen( |
+ Fts5Config *pConfig, |
+ int bCreate, |
+ Fts5Index **pp, |
+ char **pzErr |
+){ |
+ int rc = SQLITE_OK; |
+ Fts5Index *p; /* New object */ |
+ |
+ *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index)); |
+ if( rc==SQLITE_OK ){ |
+ p->pConfig = pConfig; |
+ p->nWorkUnit = FTS5_WORK_UNIT; |
+ p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName); |
+ if( p->zDataTbl && bCreate ){ |
+ rc = sqlite3Fts5CreateTable( |
+ pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr |
+ ); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5CreateTable(pConfig, "idx", |
+ "segid, term, pgno, PRIMARY KEY(segid, term)", |
+ 1, pzErr |
+ ); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5IndexReinit(p); |
+ } |
+ } |
+ } |
+ |
+ assert( rc!=SQLITE_OK || p->rc==SQLITE_OK ); |
+ if( rc ){ |
+ sqlite3Fts5IndexClose(p); |
+ *pp = 0; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen(). |
+*/ |
+static int sqlite3Fts5IndexClose(Fts5Index *p){ |
+ int rc = SQLITE_OK; |
+ if( p ){ |
+ assert( p->pReader==0 ); |
+ fts5StructureInvalidate(p); |
+ sqlite3_finalize(p->pWriter); |
+ sqlite3_finalize(p->pDeleter); |
+ sqlite3_finalize(p->pIdxWriter); |
+ sqlite3_finalize(p->pIdxDeleter); |
+ sqlite3_finalize(p->pIdxSelect); |
+ sqlite3_finalize(p->pDataVersion); |
+ sqlite3Fts5HashFree(p->pHash); |
+ sqlite3_free(p->zDataTbl); |
+ sqlite3_free(p); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Argument p points to a buffer containing utf-8 text that is n bytes in |
+** size. Return the number of bytes in the nChar character prefix of the |
+** buffer, or 0 if there are less than nChar characters in total. |
+*/ |
+static int sqlite3Fts5IndexCharlenToBytelen( |
+ const char *p, |
+ int nByte, |
+ int nChar |
+){ |
+ int n = 0; |
+ int i; |
+ for(i=0; i<nChar; i++){ |
+ if( n>=nByte ) return 0; /* Input contains fewer than nChar chars */ |
+ if( (unsigned char)p[n++]>=0xc0 ){ |
+ while( (p[n] & 0xc0)==0x80 ) n++; |
+ } |
+ } |
+ return n; |
+} |
+ |
+/* |
+** pIn is a UTF-8 encoded string, nIn bytes in size. Return the number of |
+** unicode characters in the string. |
+*/ |
+static int fts5IndexCharlen(const char *pIn, int nIn){ |
+ int nChar = 0; |
+ int i = 0; |
+ while( i<nIn ){ |
+ if( (unsigned char)pIn[i++]>=0xc0 ){ |
+ while( i<nIn && (pIn[i] & 0xc0)==0x80 ) i++; |
+ } |
+ nChar++; |
+ } |
+ return nChar; |
+} |
+ |
+/* |
+** Insert or remove data to or from the index. Each time a document is |
+** added to or removed from the index, this function is called one or more |
+** times. |
+** |
+** For an insert, it must be called once for each token in the new document. |
+** If the operation is a delete, it must be called (at least) once for each |
+** unique token in the document with an iCol value less than zero. The iPos |
+** argument is ignored for a delete. |
+*/ |
+static int sqlite3Fts5IndexWrite( |
+ Fts5Index *p, /* Index to write to */ |
+ int iCol, /* Column token appears in (-ve -> delete) */ |
+ int iPos, /* Position of token within column */ |
+ const char *pToken, int nToken /* Token to add or remove to or from index */ |
+){ |
+ int i; /* Used to iterate through indexes */ |
+ int rc = SQLITE_OK; /* Return code */ |
+ Fts5Config *pConfig = p->pConfig; |
+ |
+ assert( p->rc==SQLITE_OK ); |
+ assert( (iCol<0)==p->bDelete ); |
+ |
+ /* Add the entry to the main terms index. */ |
+ rc = sqlite3Fts5HashWrite( |
+ p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken |
+ ); |
+ |
+ for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){ |
+ const int nChar = pConfig->aPrefix[i]; |
+ int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); |
+ if( nByte ){ |
+ rc = sqlite3Fts5HashWrite(p->pHash, |
+ p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, |
+ nByte |
+ ); |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Open a new iterator to iterate though all rowid that match the |
+** specified token or token prefix. |
+*/ |
+static int sqlite3Fts5IndexQuery( |
+ Fts5Index *p, /* FTS index to query */ |
+ const char *pToken, int nToken, /* Token (or prefix) to query for */ |
+ int flags, /* Mask of FTS5INDEX_QUERY_X flags */ |
+ Fts5Colset *pColset, /* Match these columns only */ |
+ Fts5IndexIter **ppIter /* OUT: New iterator object */ |
+){ |
+ Fts5Config *pConfig = p->pConfig; |
+ Fts5Iter *pRet = 0; |
+ Fts5Buffer buf = {0, 0, 0}; |
+ |
+ /* If the QUERY_SCAN flag is set, all other flags must be clear. */ |
+ assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN ); |
+ |
+ if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ |
+ int iIdx = 0; /* Index to search */ |
+ memcpy(&buf.p[1], pToken, nToken); |
+ |
+ /* Figure out which index to search and set iIdx accordingly. If this |
+ ** is a prefix query for which there is no prefix index, set iIdx to |
+ ** greater than pConfig->nPrefix to indicate that the query will be |
+ ** satisfied by scanning multiple terms in the main index. |
+ ** |
+ ** If the QUERY_TEST_NOIDX flag was specified, then this must be a |
+ ** prefix-query. Instead of using a prefix-index (if one exists), |
+ ** evaluate the prefix query using the main FTS index. This is used |
+ ** for internal sanity checking by the integrity-check in debug |
+ ** mode only. */ |
+#ifdef SQLITE_DEBUG |
+ if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ |
+ assert( flags & FTS5INDEX_QUERY_PREFIX ); |
+ iIdx = 1+pConfig->nPrefix; |
+ }else |
+#endif |
+ if( flags & FTS5INDEX_QUERY_PREFIX ){ |
+ int nChar = fts5IndexCharlen(pToken, nToken); |
+ for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){ |
+ if( pConfig->aPrefix[iIdx-1]==nChar ) break; |
+ } |
+ } |
+ |
+ if( iIdx<=pConfig->nPrefix ){ |
+ /* Straight index lookup */ |
+ Fts5Structure *pStruct = fts5StructureRead(p); |
+ buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); |
+ if( pStruct ){ |
+ fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY, |
+ pColset, buf.p, nToken+1, -1, 0, &pRet |
+ ); |
+ fts5StructureRelease(pStruct); |
+ } |
+ }else{ |
+ /* Scan multiple terms in the main index */ |
+ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; |
+ buf.p[0] = FTS5_MAIN_PREFIX; |
+ fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet); |
+ assert( p->rc!=SQLITE_OK || pRet->pColset==0 ); |
+ fts5IterSetOutputCb(&p->rc, pRet); |
+ if( p->rc==SQLITE_OK ){ |
+ Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; |
+ if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); |
+ } |
+ } |
+ |
+ if( p->rc ){ |
+ sqlite3Fts5IterClose(&pRet->base); |
+ pRet = 0; |
+ fts5CloseReader(p); |
+ } |
+ |
+ *ppIter = &pRet->base; |
+ sqlite3Fts5BufferFree(&buf); |
+ } |
+ return fts5IndexReturn(p); |
+} |
+ |
+/* |
+** Return true if the iterator passed as the only argument is at EOF. |
+*/ |
+/* |
+** Move to the next matching rowid. |
+*/ |
+static int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ |
+ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; |
+ assert( pIter->pIndex->rc==SQLITE_OK ); |
+ fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); |
+ return fts5IndexReturn(pIter->pIndex); |
+} |
+ |
+/* |
+** Move to the next matching term/rowid. Used by the fts5vocab module. |
+*/ |
+static int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ |
+ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; |
+ Fts5Index *p = pIter->pIndex; |
+ |
+ assert( pIter->pIndex->rc==SQLITE_OK ); |
+ |
+ fts5MultiIterNext(p, pIter, 0, 0); |
+ if( p->rc==SQLITE_OK ){ |
+ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; |
+ if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){ |
+ fts5DataRelease(pSeg->pLeaf); |
+ pSeg->pLeaf = 0; |
+ pIter->base.bEof = 1; |
+ } |
+ } |
+ |
+ return fts5IndexReturn(pIter->pIndex); |
+} |
+ |
+/* |
+** Move to the next matching rowid that occurs at or after iMatch. The |
+** definition of "at or after" depends on whether this iterator iterates |
+** in ascending or descending rowid order. |
+*/ |
+static int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ |
+ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; |
+ fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); |
+ return fts5IndexReturn(pIter->pIndex); |
+} |
+ |
+/* |
+** Return the current term. |
+*/ |
+static const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ |
+ int n; |
+ const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n); |
+ *pn = n-1; |
+ return &z[1]; |
+} |
+ |
+/* |
+** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). |
+*/ |
+static void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ |
+ if( pIndexIter ){ |
+ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; |
+ Fts5Index *pIndex = pIter->pIndex; |
+ fts5MultiIterFree(pIter); |
+ fts5CloseReader(pIndex); |
+ } |
+} |
+ |
+/* |
+** Read and decode the "averages" record from the database. |
+** |
+** Parameter anSize must point to an array of size nCol, where nCol is |
+** the number of user defined columns in the FTS table. |
+*/ |
+static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){ |
+ int nCol = p->pConfig->nCol; |
+ Fts5Data *pData; |
+ |
+ *pnRow = 0; |
+ memset(anSize, 0, sizeof(i64) * nCol); |
+ pData = fts5DataRead(p, FTS5_AVERAGES_ROWID); |
+ if( p->rc==SQLITE_OK && pData->nn ){ |
+ int i = 0; |
+ int iCol; |
+ i += fts5GetVarint(&pData->p[i], (u64*)pnRow); |
+ for(iCol=0; i<pData->nn && iCol<nCol; iCol++){ |
+ i += fts5GetVarint(&pData->p[i], (u64*)&anSize[iCol]); |
+ } |
+ } |
+ |
+ fts5DataRelease(pData); |
+ return fts5IndexReturn(p); |
+} |
+ |
+/* |
+** Replace the current "averages" record with the contents of the buffer |
+** supplied as the second argument. |
+*/ |
+static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){ |
+ assert( p->rc==SQLITE_OK ); |
+ fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData); |
+ return fts5IndexReturn(p); |
+} |
+ |
+/* |
+** Return the total number of blocks this module has read from the %_data |
+** table since it was created. |
+*/ |
+static int sqlite3Fts5IndexReads(Fts5Index *p){ |
+ return p->nRead; |
+} |
+ |
+/* |
+** Set the 32-bit cookie value stored at the start of all structure |
+** records to the value passed as the second argument. |
+** |
+** Return SQLITE_OK if successful, or an SQLite error code if an error |
+** occurs. |
+*/ |
+static int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){ |
+ int rc; /* Return code */ |
+ Fts5Config *pConfig = p->pConfig; /* Configuration object */ |
+ u8 aCookie[4]; /* Binary representation of iNew */ |
+ sqlite3_blob *pBlob = 0; |
+ |
+ assert( p->rc==SQLITE_OK ); |
+ sqlite3Fts5Put32(aCookie, iNew); |
+ |
+ rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, |
+ "block", FTS5_STRUCTURE_ROWID, 1, &pBlob |
+ ); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_blob_write(pBlob, aCookie, 4, 0); |
+ rc = sqlite3_blob_close(pBlob); |
+ } |
+ |
+ return rc; |
+} |
+ |
+static int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ |
+ Fts5Structure *pStruct; |
+ pStruct = fts5StructureRead(p); |
+ fts5StructureRelease(pStruct); |
+ return fts5IndexReturn(p); |
+} |
+ |
+ |
+/************************************************************************* |
+************************************************************************** |
+** Below this point is the implementation of the integrity-check |
+** functionality. |
+*/ |
+ |
+/* |
+** Return a simple checksum value based on the arguments. |
+*/ |
+static u64 sqlite3Fts5IndexEntryCksum( |
+ i64 iRowid, |
+ int iCol, |
+ int iPos, |
+ int iIdx, |
+ const char *pTerm, |
+ int nTerm |
+){ |
+ int i; |
+ u64 ret = iRowid; |
+ ret += (ret<<3) + iCol; |
+ ret += (ret<<3) + iPos; |
+ if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX + iIdx); |
+ for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i]; |
+ return ret; |
+} |
+ |
+#ifdef SQLITE_DEBUG |
+/* |
+** This function is purely an internal test. It does not contribute to |
+** FTS functionality, or even the integrity-check, in any way. |
+** |
+** Instead, it tests that the same set of pgno/rowid combinations are |
+** visited regardless of whether the doclist-index identified by parameters |
+** iSegid/iLeaf is iterated in forwards or reverse order. |
+*/ |
+static void fts5TestDlidxReverse( |
+ Fts5Index *p, |
+ int iSegid, /* Segment id to load from */ |
+ int iLeaf /* Load doclist-index for this leaf */ |
+){ |
+ Fts5DlidxIter *pDlidx = 0; |
+ u64 cksum1 = 13; |
+ u64 cksum2 = 13; |
+ |
+ for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iLeaf); |
+ fts5DlidxIterEof(p, pDlidx)==0; |
+ fts5DlidxIterNext(p, pDlidx) |
+ ){ |
+ i64 iRowid = fts5DlidxIterRowid(pDlidx); |
+ int pgno = fts5DlidxIterPgno(pDlidx); |
+ assert( pgno>iLeaf ); |
+ cksum1 += iRowid + ((i64)pgno<<32); |
+ } |
+ fts5DlidxIterFree(pDlidx); |
+ pDlidx = 0; |
+ |
+ for(pDlidx=fts5DlidxIterInit(p, 1, iSegid, iLeaf); |
+ fts5DlidxIterEof(p, pDlidx)==0; |
+ fts5DlidxIterPrev(p, pDlidx) |
+ ){ |
+ i64 iRowid = fts5DlidxIterRowid(pDlidx); |
+ int pgno = fts5DlidxIterPgno(pDlidx); |
+ assert( fts5DlidxIterPgno(pDlidx)>iLeaf ); |
+ cksum2 += iRowid + ((i64)pgno<<32); |
+ } |
+ fts5DlidxIterFree(pDlidx); |
+ pDlidx = 0; |
+ |
+ if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT; |
+} |
+ |
+static int fts5QueryCksum( |
+ Fts5Index *p, /* Fts5 index object */ |
+ int iIdx, |
+ const char *z, /* Index key to query for */ |
+ int n, /* Size of index key in bytes */ |
+ int flags, /* Flags for Fts5IndexQuery */ |
+ u64 *pCksum /* IN/OUT: Checksum value */ |
+){ |
+ int eDetail = p->pConfig->eDetail; |
+ u64 cksum = *pCksum; |
+ Fts5IndexIter *pIter = 0; |
+ int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter); |
+ |
+ while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIter) ){ |
+ i64 rowid = pIter->iRowid; |
+ |
+ if( eDetail==FTS5_DETAIL_NONE ){ |
+ cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n); |
+ }else{ |
+ Fts5PoslistReader sReader; |
+ for(sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &sReader); |
+ sReader.bEof==0; |
+ sqlite3Fts5PoslistReaderNext(&sReader) |
+ ){ |
+ int iCol = FTS5_POS2COLUMN(sReader.iPos); |
+ int iOff = FTS5_POS2OFFSET(sReader.iPos); |
+ cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); |
+ } |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5IterNext(pIter); |
+ } |
+ } |
+ sqlite3Fts5IterClose(pIter); |
+ |
+ *pCksum = cksum; |
+ return rc; |
+} |
+ |
+ |
+/* |
+** This function is also purely an internal test. It does not contribute to |
+** FTS functionality, or even the integrity-check, in any way. |
+*/ |
+static void fts5TestTerm( |
+ Fts5Index *p, |
+ Fts5Buffer *pPrev, /* Previous term */ |
+ const char *z, int n, /* Possibly new term to test */ |
+ u64 expected, |
+ u64 *pCksum |
+){ |
+ int rc = p->rc; |
+ if( pPrev->n==0 ){ |
+ fts5BufferSet(&rc, pPrev, n, (const u8*)z); |
+ }else |
+ if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){ |
+ u64 cksum3 = *pCksum; |
+ const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */ |
+ int nTerm = pPrev->n-1; /* Size of zTerm in bytes */ |
+ int iIdx = (pPrev->p[0] - FTS5_MAIN_PREFIX); |
+ int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX); |
+ u64 ck1 = 0; |
+ u64 ck2 = 0; |
+ |
+ /* Check that the results returned for ASC and DESC queries are |
+ ** the same. If not, call this corruption. */ |
+ rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1); |
+ if( rc==SQLITE_OK ){ |
+ int f = flags|FTS5INDEX_QUERY_DESC; |
+ rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); |
+ } |
+ if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; |
+ |
+ /* If this is a prefix query, check that the results returned if the |
+ ** the index is disabled are the same. In both ASC and DESC order. |
+ ** |
+ ** This check may only be performed if the hash table is empty. This |
+ ** is because the hash table only supports a single scan query at |
+ ** a time, and the multi-iter loop from which this function is called |
+ ** is already performing such a scan. */ |
+ if( p->nPendingData==0 ){ |
+ if( iIdx>0 && rc==SQLITE_OK ){ |
+ int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; |
+ ck2 = 0; |
+ rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); |
+ if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; |
+ } |
+ if( iIdx>0 && rc==SQLITE_OK ){ |
+ int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; |
+ ck2 = 0; |
+ rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); |
+ if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; |
+ } |
+ } |
+ |
+ cksum3 ^= ck1; |
+ fts5BufferSet(&rc, pPrev, n, (const u8*)z); |
+ |
+ if( rc==SQLITE_OK && cksum3!=expected ){ |
+ rc = FTS5_CORRUPT; |
+ } |
+ *pCksum = cksum3; |
+ } |
+ p->rc = rc; |
+} |
+ |
+#else |
+# define fts5TestDlidxReverse(x,y,z) |
+# define fts5TestTerm(u,v,w,x,y,z) |
+#endif |
+ |
+/* |
+** Check that: |
+** |
+** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and |
+** contain zero terms. |
+** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and |
+** contain zero rowids. |
+*/ |
+static void fts5IndexIntegrityCheckEmpty( |
+ Fts5Index *p, |
+ Fts5StructureSegment *pSeg, /* Segment to check internal consistency */ |
+ int iFirst, |
+ int iNoRowid, |
+ int iLast |
+){ |
+ int i; |
+ |
+ /* Now check that the iter.nEmpty leaves following the current leaf |
+ ** (a) exist and (b) contain no terms. */ |
+ for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ |
+ Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); |
+ if( pLeaf ){ |
+ if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; |
+ if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; |
+ } |
+ fts5DataRelease(pLeaf); |
+ } |
+} |
+ |
+static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ |
+ int iTermOff = 0; |
+ int ii; |
+ |
+ Fts5Buffer buf1 = {0,0,0}; |
+ Fts5Buffer buf2 = {0,0,0}; |
+ |
+ ii = pLeaf->szLeaf; |
+ while( ii<pLeaf->nn && p->rc==SQLITE_OK ){ |
+ int res; |
+ int iOff; |
+ int nIncr; |
+ |
+ ii += fts5GetVarint32(&pLeaf->p[ii], nIncr); |
+ iTermOff += nIncr; |
+ iOff = iTermOff; |
+ |
+ if( iOff>=pLeaf->szLeaf ){ |
+ p->rc = FTS5_CORRUPT; |
+ }else if( iTermOff==nIncr ){ |
+ int nByte; |
+ iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); |
+ if( (iOff+nByte)>pLeaf->szLeaf ){ |
+ p->rc = FTS5_CORRUPT; |
+ }else{ |
+ fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); |
+ } |
+ }else{ |
+ int nKeep, nByte; |
+ iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep); |
+ iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); |
+ if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){ |
+ p->rc = FTS5_CORRUPT; |
+ }else{ |
+ buf1.n = nKeep; |
+ fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); |
+ } |
+ |
+ if( p->rc==SQLITE_OK ){ |
+ res = fts5BufferCompare(&buf1, &buf2); |
+ if( res<=0 ) p->rc = FTS5_CORRUPT; |
+ } |
+ } |
+ fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p); |
+ } |
+ |
+ fts5BufferFree(&buf1); |
+ fts5BufferFree(&buf2); |
+} |
+ |
+static void fts5IndexIntegrityCheckSegment( |
+ Fts5Index *p, /* FTS5 backend object */ |
+ Fts5StructureSegment *pSeg /* Segment to check internal consistency */ |
+){ |
+ Fts5Config *pConfig = p->pConfig; |
+ sqlite3_stmt *pStmt = 0; |
+ int rc2; |
+ int iIdxPrevLeaf = pSeg->pgnoFirst-1; |
+ int iDlidxPrevLeaf = pSeg->pgnoLast; |
+ |
+ if( pSeg->pgnoFirst==0 ) return; |
+ |
+ fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf( |
+ "SELECT segid, term, (pgno>>1), (pgno&1) FROM %Q.'%q_idx' WHERE segid=%d", |
+ pConfig->zDb, pConfig->zName, pSeg->iSegid |
+ )); |
+ |
+ /* Iterate through the b-tree hierarchy. */ |
+ while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ i64 iRow; /* Rowid for this leaf */ |
+ Fts5Data *pLeaf; /* Data for this leaf */ |
+ |
+ int nIdxTerm = sqlite3_column_bytes(pStmt, 1); |
+ const char *zIdxTerm = (const char*)sqlite3_column_text(pStmt, 1); |
+ int iIdxLeaf = sqlite3_column_int(pStmt, 2); |
+ int bIdxDlidx = sqlite3_column_int(pStmt, 3); |
+ |
+ /* If the leaf in question has already been trimmed from the segment, |
+ ** ignore this b-tree entry. Otherwise, load it into memory. */ |
+ if( iIdxLeaf<pSeg->pgnoFirst ) continue; |
+ iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf); |
+ pLeaf = fts5LeafRead(p, iRow); |
+ if( pLeaf==0 ) break; |
+ |
+ /* Check that the leaf contains at least one term, and that it is equal |
+ ** to or larger than the split-key in zIdxTerm. Also check that if there |
+ ** is also a rowid pointer within the leaf page header, it points to a |
+ ** location before the term. */ |
+ if( pLeaf->nn<=pLeaf->szLeaf ){ |
+ p->rc = FTS5_CORRUPT; |
+ }else{ |
+ int iOff; /* Offset of first term on leaf */ |
+ int iRowidOff; /* Offset of first rowid on leaf */ |
+ int nTerm; /* Size of term on leaf in bytes */ |
+ int res; /* Comparison of term and split-key */ |
+ |
+ iOff = fts5LeafFirstTermOff(pLeaf); |
+ iRowidOff = fts5LeafFirstRowidOff(pLeaf); |
+ if( iRowidOff>=iOff ){ |
+ p->rc = FTS5_CORRUPT; |
+ }else{ |
+ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); |
+ res = memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); |
+ if( res==0 ) res = nTerm - nIdxTerm; |
+ if( res<0 ) p->rc = FTS5_CORRUPT; |
+ } |
+ |
+ fts5IntegrityCheckPgidx(p, pLeaf); |
+ } |
+ fts5DataRelease(pLeaf); |
+ if( p->rc ) break; |
+ |
+ /* Now check that the iter.nEmpty leaves following the current leaf |
+ ** (a) exist and (b) contain no terms. */ |
+ fts5IndexIntegrityCheckEmpty( |
+ p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1 |
+ ); |
+ if( p->rc ) break; |
+ |
+ /* If there is a doclist-index, check that it looks right. */ |
+ if( bIdxDlidx ){ |
+ Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */ |
+ int iPrevLeaf = iIdxLeaf; |
+ int iSegid = pSeg->iSegid; |
+ int iPg = 0; |
+ i64 iKey; |
+ |
+ for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf); |
+ fts5DlidxIterEof(p, pDlidx)==0; |
+ fts5DlidxIterNext(p, pDlidx) |
+ ){ |
+ |
+ /* Check any rowid-less pages that occur before the current leaf. */ |
+ for(iPg=iPrevLeaf+1; iPg<fts5DlidxIterPgno(pDlidx); iPg++){ |
+ iKey = FTS5_SEGMENT_ROWID(iSegid, iPg); |
+ pLeaf = fts5DataRead(p, iKey); |
+ if( pLeaf ){ |
+ if( fts5LeafFirstRowidOff(pLeaf)!=0 ) p->rc = FTS5_CORRUPT; |
+ fts5DataRelease(pLeaf); |
+ } |
+ } |
+ iPrevLeaf = fts5DlidxIterPgno(pDlidx); |
+ |
+ /* Check that the leaf page indicated by the iterator really does |
+ ** contain the rowid suggested by the same. */ |
+ iKey = FTS5_SEGMENT_ROWID(iSegid, iPrevLeaf); |
+ pLeaf = fts5DataRead(p, iKey); |
+ if( pLeaf ){ |
+ i64 iRowid; |
+ int iRowidOff = fts5LeafFirstRowidOff(pLeaf); |
+ ASSERT_SZLEAF_OK(pLeaf); |
+ if( iRowidOff>=pLeaf->szLeaf ){ |
+ p->rc = FTS5_CORRUPT; |
+ }else{ |
+ fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); |
+ if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT; |
+ } |
+ fts5DataRelease(pLeaf); |
+ } |
+ } |
+ |
+ iDlidxPrevLeaf = iPg; |
+ fts5DlidxIterFree(pDlidx); |
+ fts5TestDlidxReverse(p, iSegid, iIdxLeaf); |
+ }else{ |
+ iDlidxPrevLeaf = pSeg->pgnoLast; |
+ /* TODO: Check there is no doclist index */ |
+ } |
+ |
+ iIdxPrevLeaf = iIdxLeaf; |
+ } |
+ |
+ rc2 = sqlite3_finalize(pStmt); |
+ if( p->rc==SQLITE_OK ) p->rc = rc2; |
+ |
+ /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */ |
+#if 0 |
+ if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){ |
+ p->rc = FTS5_CORRUPT; |
+ } |
+#endif |
+} |
+ |
+ |
+/* |
+** Run internal checks to ensure that the FTS index (a) is internally |
+** consistent and (b) contains entries for which the XOR of the checksums |
+** as calculated by sqlite3Fts5IndexEntryCksum() is cksum. |
+** |
+** Return SQLITE_CORRUPT if any of the internal checks fail, or if the |
+** checksum does not match. Return SQLITE_OK if all checks pass without |
+** error, or some other SQLite error code if another error (e.g. OOM) |
+** occurs. |
+*/ |
+static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ |
+ int eDetail = p->pConfig->eDetail; |
+ u64 cksum2 = 0; /* Checksum based on contents of indexes */ |
+ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ |
+ Fts5Iter *pIter; /* Used to iterate through entire index */ |
+ Fts5Structure *pStruct; /* Index structure */ |
+ |
+#ifdef SQLITE_DEBUG |
+ /* Used by extra internal tests only run if NDEBUG is not defined */ |
+ u64 cksum3 = 0; /* Checksum based on contents of indexes */ |
+ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ |
+#endif |
+ const int flags = FTS5INDEX_QUERY_NOOUTPUT; |
+ |
+ /* Load the FTS index structure */ |
+ pStruct = fts5StructureRead(p); |
+ |
+ /* Check that the internal nodes of each segment match the leaves */ |
+ if( pStruct ){ |
+ int iLvl, iSeg; |
+ for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ |
+ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ |
+ Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; |
+ fts5IndexIntegrityCheckSegment(p, pSeg); |
+ } |
+ } |
+ } |
+ |
+ /* The cksum argument passed to this function is a checksum calculated |
+ ** based on all expected entries in the FTS index (including prefix index |
+ ** entries). This block checks that a checksum calculated based on the |
+ ** actual contents of FTS index is identical. |
+ ** |
+ ** Two versions of the same checksum are calculated. The first (stack |
+ ** variable cksum2) based on entries extracted from the full-text index |
+ ** while doing a linear scan of each individual index in turn. |
+ ** |
+ ** As each term visited by the linear scans, a separate query for the |
+ ** same term is performed. cksum3 is calculated based on the entries |
+ ** extracted by these queries. |
+ */ |
+ for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, -1, 0, &pIter); |
+ fts5MultiIterEof(p, pIter)==0; |
+ fts5MultiIterNext(p, pIter, 0, 0) |
+ ){ |
+ int n; /* Size of term in bytes */ |
+ i64 iPos = 0; /* Position read from poslist */ |
+ int iOff = 0; /* Offset within poslist */ |
+ i64 iRowid = fts5MultiIterRowid(pIter); |
+ char *z = (char*)fts5MultiIterTerm(pIter, &n); |
+ |
+ /* If this is a new term, query for it. Update cksum3 with the results. */ |
+ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); |
+ |
+ if( eDetail==FTS5_DETAIL_NONE ){ |
+ if( 0==fts5MultiIterIsEmpty(p, pIter) ){ |
+ cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); |
+ } |
+ }else{ |
+ poslist.n = 0; |
+ fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); |
+ while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ |
+ int iCol = FTS5_POS2COLUMN(iPos); |
+ int iTokOff = FTS5_POS2OFFSET(iPos); |
+ cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); |
+ } |
+ } |
+ } |
+ fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); |
+ |
+ fts5MultiIterFree(pIter); |
+ if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; |
+ |
+ fts5StructureRelease(pStruct); |
+#ifdef SQLITE_DEBUG |
+ fts5BufferFree(&term); |
+#endif |
+ fts5BufferFree(&poslist); |
+ return fts5IndexReturn(p); |
+} |
+ |
+/************************************************************************* |
+************************************************************************** |
+** Below this point is the implementation of the fts5_decode() scalar |
+** function only. |
+*/ |
+ |
+/* |
+** Decode a segment-data rowid from the %_data table. This function is |
+** the opposite of macro FTS5_SEGMENT_ROWID(). |
+*/ |
+static void fts5DecodeRowid( |
+ i64 iRowid, /* Rowid from %_data table */ |
+ int *piSegid, /* OUT: Segment id */ |
+ int *pbDlidx, /* OUT: Dlidx flag */ |
+ int *piHeight, /* OUT: Height */ |
+ int *piPgno /* OUT: Page number */ |
+){ |
+ *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1)); |
+ iRowid >>= FTS5_DATA_PAGE_B; |
+ |
+ *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1)); |
+ iRowid >>= FTS5_DATA_HEIGHT_B; |
+ |
+ *pbDlidx = (int)(iRowid & 0x0001); |
+ iRowid >>= FTS5_DATA_DLI_B; |
+ |
+ *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); |
+} |
+ |
+static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ |
+ int iSegid, iHeight, iPgno, bDlidx; /* Rowid compenents */ |
+ fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno); |
+ |
+ if( iSegid==0 ){ |
+ if( iKey==FTS5_AVERAGES_ROWID ){ |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{averages} "); |
+ }else{ |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{structure}"); |
+ } |
+ } |
+ else{ |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}", |
+ bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno |
+ ); |
+ } |
+} |
+ |
+static void fts5DebugStructure( |
+ int *pRc, /* IN/OUT: error code */ |
+ Fts5Buffer *pBuf, |
+ Fts5Structure *p |
+){ |
+ int iLvl, iSeg; /* Iterate through levels, segments */ |
+ |
+ for(iLvl=0; iLvl<p->nLevel; iLvl++){ |
+ Fts5StructureLevel *pLvl = &p->aLevel[iLvl]; |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, |
+ " {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg |
+ ); |
+ for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){ |
+ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", |
+ pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast |
+ ); |
+ } |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); |
+ } |
+} |
+ |
+/* |
+** This is part of the fts5_decode() debugging aid. |
+** |
+** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This |
+** function appends a human-readable representation of the same object |
+** to the buffer passed as the second argument. |
+*/ |
+static void fts5DecodeStructure( |
+ int *pRc, /* IN/OUT: error code */ |
+ Fts5Buffer *pBuf, |
+ const u8 *pBlob, int nBlob |
+){ |
+ int rc; /* Return code */ |
+ Fts5Structure *p = 0; /* Decoded structure object */ |
+ |
+ rc = fts5StructureDecode(pBlob, nBlob, 0, &p); |
+ if( rc!=SQLITE_OK ){ |
+ *pRc = rc; |
+ return; |
+ } |
+ |
+ fts5DebugStructure(pRc, pBuf, p); |
+ fts5StructureRelease(p); |
+} |
+ |
+/* |
+** This is part of the fts5_decode() debugging aid. |
+** |
+** Arguments pBlob/nBlob contain an "averages" record. This function |
+** appends a human-readable representation of record to the buffer passed |
+** as the second argument. |
+*/ |
+static void fts5DecodeAverages( |
+ int *pRc, /* IN/OUT: error code */ |
+ Fts5Buffer *pBuf, |
+ const u8 *pBlob, int nBlob |
+){ |
+ int i = 0; |
+ const char *zSpace = ""; |
+ |
+ while( i<nBlob ){ |
+ u64 iVal; |
+ i += sqlite3Fts5GetVarint(&pBlob[i], &iVal); |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "%s%d", zSpace, (int)iVal); |
+ zSpace = " "; |
+ } |
+} |
+ |
+/* |
+** Buffer (a/n) is assumed to contain a list of serialized varints. Read |
+** each varint and append its string representation to buffer pBuf. Return |
+** after either the input buffer is exhausted or a 0 value is read. |
+** |
+** The return value is the number of bytes read from the input buffer. |
+*/ |
+static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ |
+ int iOff = 0; |
+ while( iOff<n ){ |
+ int iVal; |
+ iOff += fts5GetVarint32(&a[iOff], iVal); |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal); |
+ } |
+ return iOff; |
+} |
+ |
+/* |
+** The start of buffer (a/n) contains the start of a doclist. The doclist |
+** may or may not finish within the buffer. This function appends a text |
+** representation of the part of the doclist that is present to buffer |
+** pBuf. |
+** |
+** The return value is the number of bytes read from the input buffer. |
+*/ |
+static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ |
+ i64 iDocid = 0; |
+ int iOff = 0; |
+ |
+ if( n>0 ){ |
+ iOff = sqlite3Fts5GetVarint(a, (u64*)&iDocid); |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); |
+ } |
+ while( iOff<n ){ |
+ int nPos; |
+ int bDel; |
+ iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDel); |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " nPos=%d%s", nPos, bDel?"*":""); |
+ iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos)); |
+ if( iOff<n ){ |
+ i64 iDelta; |
+ iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDelta); |
+ iDocid += iDelta; |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); |
+ } |
+ } |
+ |
+ return iOff; |
+} |
+ |
+/* |
+** This function is part of the fts5_decode() debugging function. It is |
+** only ever used with detail=none tables. |
+** |
+** Buffer (pData/nData) contains a doclist in the format used by detail=none |
+** tables. This function appends a human-readable version of that list to |
+** buffer pBuf. |
+** |
+** If *pRc is other than SQLITE_OK when this function is called, it is a |
+** no-op. If an OOM or other error occurs within this function, *pRc is |
+** set to an SQLite error code before returning. The final state of buffer |
+** pBuf is undefined in this case. |
+*/ |
+static void fts5DecodeRowidList( |
+ int *pRc, /* IN/OUT: Error code */ |
+ Fts5Buffer *pBuf, /* Buffer to append text to */ |
+ const u8 *pData, int nData /* Data to decode list-of-rowids from */ |
+){ |
+ int i = 0; |
+ i64 iRowid = 0; |
+ |
+ while( i<nData ){ |
+ const char *zApp = ""; |
+ u64 iVal; |
+ i += sqlite3Fts5GetVarint(&pData[i], &iVal); |
+ iRowid += iVal; |
+ |
+ if( i<nData && pData[i]==0x00 ){ |
+ i++; |
+ if( i<nData && pData[i]==0x00 ){ |
+ i++; |
+ zApp = "+"; |
+ }else{ |
+ zApp = "*"; |
+ } |
+ } |
+ |
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp); |
+ } |
+} |
+ |
+/* |
+** The implementation of user-defined scalar function fts5_decode(). |
+*/ |
+static void fts5DecodeFunction( |
+ sqlite3_context *pCtx, /* Function call context */ |
+ int nArg, /* Number of args (always 2) */ |
+ sqlite3_value **apVal /* Function arguments */ |
+){ |
+ i64 iRowid; /* Rowid for record being decoded */ |
+ int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */ |
+ const u8 *aBlob; int n; /* Record to decode */ |
+ u8 *a = 0; |
+ Fts5Buffer s; /* Build up text to return here */ |
+ int rc = SQLITE_OK; /* Return code */ |
+ int nSpace = 0; |
+ int eDetailNone = (sqlite3_user_data(pCtx)!=0); |
+ |
+ assert( nArg==2 ); |
+ UNUSED_PARAM(nArg); |
+ memset(&s, 0, sizeof(Fts5Buffer)); |
+ iRowid = sqlite3_value_int64(apVal[0]); |
+ |
+ /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[] |
+ ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents |
+ ** buffer overreads even if the record is corrupt. */ |
+ n = sqlite3_value_bytes(apVal[1]); |
+ aBlob = sqlite3_value_blob(apVal[1]); |
+ nSpace = n + FTS5_DATA_ZERO_PADDING; |
+ a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); |
+ if( a==0 ) goto decode_out; |
+ memcpy(a, aBlob, n); |
+ |
+ |
+ fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno); |
+ |
+ fts5DebugRowid(&rc, &s, iRowid); |
+ if( bDlidx ){ |
+ Fts5Data dlidx; |
+ Fts5DlidxLvl lvl; |
+ |
+ dlidx.p = a; |
+ dlidx.nn = n; |
+ |
+ memset(&lvl, 0, sizeof(Fts5DlidxLvl)); |
+ lvl.pData = &dlidx; |
+ lvl.iLeafPgno = iPgno; |
+ |
+ for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){ |
+ sqlite3Fts5BufferAppendPrintf(&rc, &s, |
+ " %d(%lld)", lvl.iLeafPgno, lvl.iRowid |
+ ); |
+ } |
+ }else if( iSegid==0 ){ |
+ if( iRowid==FTS5_AVERAGES_ROWID ){ |
+ fts5DecodeAverages(&rc, &s, a, n); |
+ }else{ |
+ fts5DecodeStructure(&rc, &s, a, n); |
+ } |
+ }else if( eDetailNone ){ |
+ Fts5Buffer term; /* Current term read from page */ |
+ int szLeaf; |
+ int iPgidxOff = szLeaf = fts5GetU16(&a[2]); |
+ int iTermOff; |
+ int nKeep = 0; |
+ int iOff; |
+ |
+ memset(&term, 0, sizeof(Fts5Buffer)); |
+ |
+ /* Decode any entries that occur before the first term. */ |
+ if( szLeaf<n ){ |
+ iPgidxOff += fts5GetVarint32(&a[iPgidxOff], iTermOff); |
+ }else{ |
+ iTermOff = szLeaf; |
+ } |
+ fts5DecodeRowidList(&rc, &s, &a[4], iTermOff-4); |
+ |
+ iOff = iTermOff; |
+ while( iOff<szLeaf ){ |
+ int nAppend; |
+ |
+ /* Read the term data for the next term*/ |
+ iOff += fts5GetVarint32(&a[iOff], nAppend); |
+ term.n = nKeep; |
+ fts5BufferAppendBlob(&rc, &term, nAppend, &a[iOff]); |
+ sqlite3Fts5BufferAppendPrintf( |
+ &rc, &s, " term=%.*s", term.n, (const char*)term.p |
+ ); |
+ iOff += nAppend; |
+ |
+ /* Figure out where the doclist for this term ends */ |
+ if( iPgidxOff<n ){ |
+ int nIncr; |
+ iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nIncr); |
+ iTermOff += nIncr; |
+ }else{ |
+ iTermOff = szLeaf; |
+ } |
+ |
+ fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff); |
+ iOff = iTermOff; |
+ if( iOff<szLeaf ){ |
+ iOff += fts5GetVarint32(&a[iOff], nKeep); |
+ } |
+ } |
+ |
+ fts5BufferFree(&term); |
+ }else{ |
+ Fts5Buffer term; /* Current term read from page */ |
+ int szLeaf; /* Offset of pgidx in a[] */ |
+ int iPgidxOff; |
+ int iPgidxPrev = 0; /* Previous value read from pgidx */ |
+ int iTermOff = 0; |
+ int iRowidOff = 0; |
+ int iOff; |
+ int nDoclist; |
+ |
+ memset(&term, 0, sizeof(Fts5Buffer)); |
+ |
+ if( n<4 ){ |
+ sqlite3Fts5BufferSet(&rc, &s, 7, (const u8*)"corrupt"); |
+ goto decode_out; |
+ }else{ |
+ iRowidOff = fts5GetU16(&a[0]); |
+ iPgidxOff = szLeaf = fts5GetU16(&a[2]); |
+ if( iPgidxOff<n ){ |
+ fts5GetVarint32(&a[iPgidxOff], iTermOff); |
+ } |
+ } |
+ |
+ /* Decode the position list tail at the start of the page */ |
+ if( iRowidOff!=0 ){ |
+ iOff = iRowidOff; |
+ }else if( iTermOff!=0 ){ |
+ iOff = iTermOff; |
+ }else{ |
+ iOff = szLeaf; |
+ } |
+ fts5DecodePoslist(&rc, &s, &a[4], iOff-4); |
+ |
+ /* Decode any more doclist data that appears on the page before the |
+ ** first term. */ |
+ nDoclist = (iTermOff ? iTermOff : szLeaf) - iOff; |
+ fts5DecodeDoclist(&rc, &s, &a[iOff], nDoclist); |
+ |
+ while( iPgidxOff<n ){ |
+ int bFirst = (iPgidxOff==szLeaf); /* True for first term on page */ |
+ int nByte; /* Bytes of data */ |
+ int iEnd; |
+ |
+ iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nByte); |
+ iPgidxPrev += nByte; |
+ iOff = iPgidxPrev; |
+ |
+ if( iPgidxOff<n ){ |
+ fts5GetVarint32(&a[iPgidxOff], nByte); |
+ iEnd = iPgidxPrev + nByte; |
+ }else{ |
+ iEnd = szLeaf; |
+ } |
+ |
+ if( bFirst==0 ){ |
+ iOff += fts5GetVarint32(&a[iOff], nByte); |
+ term.n = nByte; |
+ } |
+ iOff += fts5GetVarint32(&a[iOff], nByte); |
+ fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]); |
+ iOff += nByte; |
+ |
+ sqlite3Fts5BufferAppendPrintf( |
+ &rc, &s, " term=%.*s", term.n, (const char*)term.p |
+ ); |
+ iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], iEnd-iOff); |
+ } |
+ |
+ fts5BufferFree(&term); |
+ } |
+ |
+ decode_out: |
+ sqlite3_free(a); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT); |
+ }else{ |
+ sqlite3_result_error_code(pCtx, rc); |
+ } |
+ fts5BufferFree(&s); |
+} |
+ |
+/* |
+** The implementation of user-defined scalar function fts5_rowid(). |
+*/ |
+static void fts5RowidFunction( |
+ sqlite3_context *pCtx, /* Function call context */ |
+ int nArg, /* Number of args (always 2) */ |
+ sqlite3_value **apVal /* Function arguments */ |
+){ |
+ const char *zArg; |
+ if( nArg==0 ){ |
+ sqlite3_result_error(pCtx, "should be: fts5_rowid(subject, ....)", -1); |
+ }else{ |
+ zArg = (const char*)sqlite3_value_text(apVal[0]); |
+ if( 0==sqlite3_stricmp(zArg, "segment") ){ |
+ i64 iRowid; |
+ int segid, pgno; |
+ if( nArg!=3 ){ |
+ sqlite3_result_error(pCtx, |
+ "should be: fts5_rowid('segment', segid, pgno))", -1 |
+ ); |
+ }else{ |
+ segid = sqlite3_value_int(apVal[1]); |
+ pgno = sqlite3_value_int(apVal[2]); |
+ iRowid = FTS5_SEGMENT_ROWID(segid, pgno); |
+ sqlite3_result_int64(pCtx, iRowid); |
+ } |
+ }else{ |
+ sqlite3_result_error(pCtx, |
+ "first arg to fts5_rowid() must be 'segment'" , -1 |
+ ); |
+ } |
+ } |
+} |
+ |
+/* |
+** This is called as part of registering the FTS5 module with database |
+** connection db. It registers several user-defined scalar functions useful |
+** with FTS5. |
+** |
+** If successful, SQLITE_OK is returned. If an error occurs, some other |
+** SQLite error code is returned instead. |
+*/ |
+static int sqlite3Fts5IndexInit(sqlite3 *db){ |
+ int rc = sqlite3_create_function( |
+ db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 |
+ ); |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3_create_function( |
+ db, "fts5_decode_none", 2, |
+ SQLITE_UTF8, (void*)db, fts5DecodeFunction, 0, 0 |
+ ); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3_create_function( |
+ db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 |
+ ); |
+ } |
+ return rc; |
+} |
+ |
+ |
+static int sqlite3Fts5IndexReset(Fts5Index *p){ |
+ assert( p->pStruct==0 || p->iStructVersion!=0 ); |
+ if( fts5IndexDataVersion(p)!=p->iStructVersion ){ |
+ fts5StructureInvalidate(p); |
+ } |
+ return fts5IndexReturn(p); |
+} |
+ |
+/* |
+** 2014 Jun 09 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** This is an SQLite module implementing full-text search. |
+*/ |
+ |
+ |
+/* #include "fts5Int.h" */ |
+ |
+/* |
+** This variable is set to false when running tests for which the on disk |
+** structures should not be corrupt. Otherwise, true. If it is false, extra |
+** assert() conditions in the fts5 code are activated - conditions that are |
+** only true if it is guaranteed that the fts5 database is not corrupt. |
+*/ |
+SQLITE_API int sqlite3_fts5_may_be_corrupt = 1; |
+ |
+ |
+typedef struct Fts5Auxdata Fts5Auxdata; |
+typedef struct Fts5Auxiliary Fts5Auxiliary; |
+typedef struct Fts5Cursor Fts5Cursor; |
+typedef struct Fts5Sorter Fts5Sorter; |
+typedef struct Fts5Table Fts5Table; |
+typedef struct Fts5TokenizerModule Fts5TokenizerModule; |
+ |
+/* |
+** NOTES ON TRANSACTIONS: |
+** |
+** SQLite invokes the following virtual table methods as transactions are |
+** opened and closed by the user: |
+** |
+** xBegin(): Start of a new transaction. |
+** xSync(): Initial part of two-phase commit. |
+** xCommit(): Final part of two-phase commit. |
+** xRollback(): Rollback the transaction. |
+** |
+** Anything that is required as part of a commit that may fail is performed |
+** in the xSync() callback. Current versions of SQLite ignore any errors |
+** returned by xCommit(). |
+** |
+** And as sub-transactions are opened/closed: |
+** |
+** xSavepoint(int S): Open savepoint S. |
+** xRelease(int S): Commit and close savepoint S. |
+** xRollbackTo(int S): Rollback to start of savepoint S. |
+** |
+** During a write-transaction the fts5_index.c module may cache some data |
+** in-memory. It is flushed to disk whenever xSync(), xRelease() or |
+** xSavepoint() is called. And discarded whenever xRollback() or xRollbackTo() |
+** is called. |
+** |
+** Additionally, if SQLITE_DEBUG is defined, an instance of the following |
+** structure is used to record the current transaction state. This information |
+** is not required, but it is used in the assert() statements executed by |
+** function fts5CheckTransactionState() (see below). |
+*/ |
+struct Fts5TransactionState { |
+ int eState; /* 0==closed, 1==open, 2==synced */ |
+ int iSavepoint; /* Number of open savepoints (0 -> none) */ |
+}; |
+ |
+/* |
+** A single object of this type is allocated when the FTS5 module is |
+** registered with a database handle. It is used to store pointers to |
+** all registered FTS5 extensions - tokenizers and auxiliary functions. |
+*/ |
+struct Fts5Global { |
+ fts5_api api; /* User visible part of object (see fts5.h) */ |
+ sqlite3 *db; /* Associated database connection */ |
+ i64 iNextId; /* Used to allocate unique cursor ids */ |
+ Fts5Auxiliary *pAux; /* First in list of all aux. functions */ |
+ Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */ |
+ Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */ |
+ Fts5Cursor *pCsr; /* First in list of all open cursors */ |
+}; |
+ |
+/* |
+** Each auxiliary function registered with the FTS5 module is represented |
+** by an object of the following type. All such objects are stored as part |
+** of the Fts5Global.pAux list. |
+*/ |
+struct Fts5Auxiliary { |
+ Fts5Global *pGlobal; /* Global context for this function */ |
+ char *zFunc; /* Function name (nul-terminated) */ |
+ void *pUserData; /* User-data pointer */ |
+ fts5_extension_function xFunc; /* Callback function */ |
+ void (*xDestroy)(void*); /* Destructor function */ |
+ Fts5Auxiliary *pNext; /* Next registered auxiliary function */ |
+}; |
+ |
+/* |
+** Each tokenizer module registered with the FTS5 module is represented |
+** by an object of the following type. All such objects are stored as part |
+** of the Fts5Global.pTok list. |
+*/ |
+struct Fts5TokenizerModule { |
+ char *zName; /* Name of tokenizer */ |
+ void *pUserData; /* User pointer passed to xCreate() */ |
+ fts5_tokenizer x; /* Tokenizer functions */ |
+ void (*xDestroy)(void*); /* Destructor function */ |
+ Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ |
+}; |
+ |
+/* |
+** Virtual-table object. |
+*/ |
+struct Fts5Table { |
+ sqlite3_vtab base; /* Base class used by SQLite core */ |
+ Fts5Config *pConfig; /* Virtual table configuration */ |
+ Fts5Index *pIndex; /* Full-text index */ |
+ Fts5Storage *pStorage; /* Document store */ |
+ Fts5Global *pGlobal; /* Global (connection wide) data */ |
+ Fts5Cursor *pSortCsr; /* Sort data from this cursor */ |
+#ifdef SQLITE_DEBUG |
+ struct Fts5TransactionState ts; |
+#endif |
+}; |
+ |
+struct Fts5MatchPhrase { |
+ Fts5Buffer *pPoslist; /* Pointer to current poslist */ |
+ int nTerm; /* Size of phrase in terms */ |
+}; |
+ |
+/* |
+** pStmt: |
+** SELECT rowid, <fts> FROM <fts> ORDER BY +rank; |
+** |
+** aIdx[]: |
+** There is one entry in the aIdx[] array for each phrase in the query, |
+** the value of which is the offset within aPoslist[] following the last |
+** byte of the position list for the corresponding phrase. |
+*/ |
+struct Fts5Sorter { |
+ sqlite3_stmt *pStmt; |
+ i64 iRowid; /* Current rowid */ |
+ const u8 *aPoslist; /* Position lists for current row */ |
+ int nIdx; /* Number of entries in aIdx[] */ |
+ int aIdx[1]; /* Offsets into aPoslist for current row */ |
+}; |
+ |
+ |
+/* |
+** Virtual-table cursor object. |
+** |
+** iSpecial: |
+** If this is a 'special' query (refer to function fts5SpecialMatch()), |
+** then this variable contains the result of the query. |
+** |
+** iFirstRowid, iLastRowid: |
+** These variables are only used for FTS5_PLAN_MATCH cursors. Assuming the |
+** cursor iterates in ascending order of rowids, iFirstRowid is the lower |
+** limit of rowids to return, and iLastRowid the upper. In other words, the |
+** WHERE clause in the user's query might have been: |
+** |
+** <tbl> MATCH <expr> AND rowid BETWEEN $iFirstRowid AND $iLastRowid |
+** |
+** If the cursor iterates in descending order of rowid, iFirstRowid |
+** is the upper limit (i.e. the "first" rowid visited) and iLastRowid |
+** the lower. |
+*/ |
+struct Fts5Cursor { |
+ sqlite3_vtab_cursor base; /* Base class used by SQLite core */ |
+ Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ |
+ int *aColumnSize; /* Values for xColumnSize() */ |
+ i64 iCsrId; /* Cursor id */ |
+ |
+ /* Zero from this point onwards on cursor reset */ |
+ int ePlan; /* FTS5_PLAN_XXX value */ |
+ int bDesc; /* True for "ORDER BY rowid DESC" queries */ |
+ i64 iFirstRowid; /* Return no rowids earlier than this */ |
+ i64 iLastRowid; /* Return no rowids later than this */ |
+ sqlite3_stmt *pStmt; /* Statement used to read %_content */ |
+ Fts5Expr *pExpr; /* Expression for MATCH queries */ |
+ Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */ |
+ int csrflags; /* Mask of cursor flags (see below) */ |
+ i64 iSpecial; /* Result of special query */ |
+ |
+ /* "rank" function. Populated on demand from vtab.xColumn(). */ |
+ char *zRank; /* Custom rank function */ |
+ char *zRankArgs; /* Custom rank function args */ |
+ Fts5Auxiliary *pRank; /* Rank callback (or NULL) */ |
+ int nRankArg; /* Number of trailing arguments for rank() */ |
+ sqlite3_value **apRankArg; /* Array of trailing arguments */ |
+ sqlite3_stmt *pRankArgStmt; /* Origin of objects in apRankArg[] */ |
+ |
+ /* Auxiliary data storage */ |
+ Fts5Auxiliary *pAux; /* Currently executing extension function */ |
+ Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ |
+ |
+ /* Cache used by auxiliary functions xInst() and xInstCount() */ |
+ Fts5PoslistReader *aInstIter; /* One for each phrase */ |
+ int nInstAlloc; /* Size of aInst[] array (entries / 3) */ |
+ int nInstCount; /* Number of phrase instances */ |
+ int *aInst; /* 3 integers per phrase instance */ |
+}; |
+ |
+/* |
+** Bits that make up the "idxNum" parameter passed indirectly by |
+** xBestIndex() to xFilter(). |
+*/ |
+#define FTS5_BI_MATCH 0x0001 /* <tbl> MATCH ? */ |
+#define FTS5_BI_RANK 0x0002 /* rank MATCH ? */ |
+#define FTS5_BI_ROWID_EQ 0x0004 /* rowid == ? */ |
+#define FTS5_BI_ROWID_LE 0x0008 /* rowid <= ? */ |
+#define FTS5_BI_ROWID_GE 0x0010 /* rowid >= ? */ |
+ |
+#define FTS5_BI_ORDER_RANK 0x0020 |
+#define FTS5_BI_ORDER_ROWID 0x0040 |
+#define FTS5_BI_ORDER_DESC 0x0080 |
+ |
+/* |
+** Values for Fts5Cursor.csrflags |
+*/ |
+#define FTS5CSR_EOF 0x01 |
+#define FTS5CSR_REQUIRE_CONTENT 0x02 |
+#define FTS5CSR_REQUIRE_DOCSIZE 0x04 |
+#define FTS5CSR_REQUIRE_INST 0x08 |
+#define FTS5CSR_FREE_ZRANK 0x10 |
+#define FTS5CSR_REQUIRE_RESEEK 0x20 |
+#define FTS5CSR_REQUIRE_POSLIST 0x40 |
+ |
+#define BitFlagAllTest(x,y) (((x) & (y))==(y)) |
+#define BitFlagTest(x,y) (((x) & (y))!=0) |
+ |
+ |
+/* |
+** Macros to Set(), Clear() and Test() cursor flags. |
+*/ |
+#define CsrFlagSet(pCsr, flag) ((pCsr)->csrflags |= (flag)) |
+#define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag)) |
+#define CsrFlagTest(pCsr, flag) ((pCsr)->csrflags & (flag)) |
+ |
+struct Fts5Auxdata { |
+ Fts5Auxiliary *pAux; /* Extension to which this belongs */ |
+ void *pPtr; /* Pointer value */ |
+ void(*xDelete)(void*); /* Destructor */ |
+ Fts5Auxdata *pNext; /* Next object in linked list */ |
+}; |
+ |
+#ifdef SQLITE_DEBUG |
+#define FTS5_BEGIN 1 |
+#define FTS5_SYNC 2 |
+#define FTS5_COMMIT 3 |
+#define FTS5_ROLLBACK 4 |
+#define FTS5_SAVEPOINT 5 |
+#define FTS5_RELEASE 6 |
+#define FTS5_ROLLBACKTO 7 |
+static void fts5CheckTransactionState(Fts5Table *p, int op, int iSavepoint){ |
+ switch( op ){ |
+ case FTS5_BEGIN: |
+ assert( p->ts.eState==0 ); |
+ p->ts.eState = 1; |
+ p->ts.iSavepoint = -1; |
+ break; |
+ |
+ case FTS5_SYNC: |
+ assert( p->ts.eState==1 ); |
+ p->ts.eState = 2; |
+ break; |
+ |
+ case FTS5_COMMIT: |
+ assert( p->ts.eState==2 ); |
+ p->ts.eState = 0; |
+ break; |
+ |
+ case FTS5_ROLLBACK: |
+ assert( p->ts.eState==1 || p->ts.eState==2 || p->ts.eState==0 ); |
+ p->ts.eState = 0; |
+ break; |
+ |
+ case FTS5_SAVEPOINT: |
+ assert( p->ts.eState==1 ); |
+ assert( iSavepoint>=0 ); |
+ assert( iSavepoint>p->ts.iSavepoint ); |
+ p->ts.iSavepoint = iSavepoint; |
+ break; |
+ |
+ case FTS5_RELEASE: |
+ assert( p->ts.eState==1 ); |
+ assert( iSavepoint>=0 ); |
+ assert( iSavepoint<=p->ts.iSavepoint ); |
+ p->ts.iSavepoint = iSavepoint-1; |
+ break; |
+ |
+ case FTS5_ROLLBACKTO: |
+ assert( p->ts.eState==1 ); |
+ assert( iSavepoint>=0 ); |
+ assert( iSavepoint<=p->ts.iSavepoint ); |
+ p->ts.iSavepoint = iSavepoint; |
+ break; |
+ } |
+} |
+#else |
+# define fts5CheckTransactionState(x,y,z) |
+#endif |
+ |
+/* |
+** Return true if pTab is a contentless table. |
+*/ |
+static int fts5IsContentless(Fts5Table *pTab){ |
+ return pTab->pConfig->eContent==FTS5_CONTENT_NONE; |
+} |
+ |
+/* |
+** Delete a virtual table handle allocated by fts5InitVtab(). |
+*/ |
+static void fts5FreeVtab(Fts5Table *pTab){ |
+ if( pTab ){ |
+ sqlite3Fts5IndexClose(pTab->pIndex); |
+ sqlite3Fts5StorageClose(pTab->pStorage); |
+ sqlite3Fts5ConfigFree(pTab->pConfig); |
+ sqlite3_free(pTab); |
+ } |
+} |
+ |
+/* |
+** The xDisconnect() virtual table method. |
+*/ |
+static int fts5DisconnectMethod(sqlite3_vtab *pVtab){ |
+ fts5FreeVtab((Fts5Table*)pVtab); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** The xDestroy() virtual table method. |
+*/ |
+static int fts5DestroyMethod(sqlite3_vtab *pVtab){ |
+ Fts5Table *pTab = (Fts5Table*)pVtab; |
+ int rc = sqlite3Fts5DropAll(pTab->pConfig); |
+ if( rc==SQLITE_OK ){ |
+ fts5FreeVtab((Fts5Table*)pVtab); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** This function is the implementation of both the xConnect and xCreate |
+** methods of the FTS3 virtual table. |
+** |
+** The argv[] array contains the following: |
+** |
+** argv[0] -> module name ("fts5") |
+** argv[1] -> database name |
+** argv[2] -> table name |
+** argv[...] -> "column name" and other module argument fields. |
+*/ |
+static int fts5InitVtab( |
+ int bCreate, /* True for xCreate, false for xConnect */ |
+ sqlite3 *db, /* The SQLite database connection */ |
+ void *pAux, /* Hash table containing tokenizers */ |
+ int argc, /* Number of elements in argv array */ |
+ const char * const *argv, /* xCreate/xConnect argument array */ |
+ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ |
+ char **pzErr /* Write any error message here */ |
+){ |
+ Fts5Global *pGlobal = (Fts5Global*)pAux; |
+ const char **azConfig = (const char**)argv; |
+ int rc = SQLITE_OK; /* Return code */ |
+ Fts5Config *pConfig = 0; /* Results of parsing argc/argv */ |
+ Fts5Table *pTab = 0; /* New virtual table object */ |
+ |
+ /* Allocate the new vtab object and parse the configuration */ |
+ pTab = (Fts5Table*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Table)); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr); |
+ assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 ); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ pTab->pConfig = pConfig; |
+ pTab->pGlobal = pGlobal; |
+ } |
+ |
+ /* Open the index sub-system */ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->pIndex, pzErr); |
+ } |
+ |
+ /* Open the storage sub-system */ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5StorageOpen( |
+ pConfig, pTab->pIndex, bCreate, &pTab->pStorage, pzErr |
+ ); |
+ } |
+ |
+ /* Call sqlite3_declare_vtab() */ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5ConfigDeclareVtab(pConfig); |
+ } |
+ |
+ /* Load the initial configuration */ |
+ if( rc==SQLITE_OK ){ |
+ assert( pConfig->pzErrmsg==0 ); |
+ pConfig->pzErrmsg = pzErr; |
+ rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex); |
+ sqlite3Fts5IndexRollback(pTab->pIndex); |
+ pConfig->pzErrmsg = 0; |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ fts5FreeVtab(pTab); |
+ pTab = 0; |
+ }else if( bCreate ){ |
+ fts5CheckTransactionState(pTab, FTS5_BEGIN, 0); |
+ } |
+ *ppVTab = (sqlite3_vtab*)pTab; |
+ return rc; |
+} |
+ |
+/* |
+** The xConnect() and xCreate() methods for the virtual table. All the |
+** work is done in function fts5InitVtab(). |
+*/ |
+static int fts5ConnectMethod( |
+ sqlite3 *db, /* Database connection */ |
+ void *pAux, /* Pointer to tokenizer hash table */ |
+ int argc, /* Number of elements in argv array */ |
+ const char * const *argv, /* xCreate/xConnect argument array */ |
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ |
+ char **pzErr /* OUT: sqlite3_malloc'd error message */ |
+){ |
+ return fts5InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); |
+} |
+static int fts5CreateMethod( |
+ sqlite3 *db, /* Database connection */ |
+ void *pAux, /* Pointer to tokenizer hash table */ |
+ int argc, /* Number of elements in argv array */ |
+ const char * const *argv, /* xCreate/xConnect argument array */ |
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ |
+ char **pzErr /* OUT: sqlite3_malloc'd error message */ |
+){ |
+ return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); |
+} |
+ |
+/* |
+** The different query plans. |
+*/ |
+#define FTS5_PLAN_MATCH 1 /* (<tbl> MATCH ?) */ |
+#define FTS5_PLAN_SOURCE 2 /* A source cursor for SORTED_MATCH */ |
+#define FTS5_PLAN_SPECIAL 3 /* An internal query */ |
+#define FTS5_PLAN_SORTED_MATCH 4 /* (<tbl> MATCH ? ORDER BY rank) */ |
+#define FTS5_PLAN_SCAN 5 /* No usable constraint */ |
+#define FTS5_PLAN_ROWID 6 /* (rowid = ?) */ |
+ |
+/* |
+** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this |
+** extension is currently being used by a version of SQLite too old to |
+** support index-info flags. In that case this function is a no-op. |
+*/ |
+static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ |
+#if SQLITE_VERSION_NUMBER>=3008012 |
+#ifndef SQLITE_CORE |
+ if( sqlite3_libversion_number()>=3008012 ) |
+#endif |
+ { |
+ pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; |
+ } |
+#endif |
+} |
+ |
+/* |
+** Implementation of the xBestIndex method for FTS5 tables. Within the |
+** WHERE constraint, it searches for the following: |
+** |
+** 1. A MATCH constraint against the special column. |
+** 2. A MATCH constraint against the "rank" column. |
+** 3. An == constraint against the rowid column. |
+** 4. A < or <= constraint against the rowid column. |
+** 5. A > or >= constraint against the rowid column. |
+** |
+** Within the ORDER BY, either: |
+** |
+** 5. ORDER BY rank [ASC|DESC] |
+** 6. ORDER BY rowid [ASC|DESC] |
+** |
+** Costs are assigned as follows: |
+** |
+** a) If an unusable MATCH operator is present in the WHERE clause, the |
+** cost is unconditionally set to 1e50 (a really big number). |
+** |
+** a) If a MATCH operator is present, the cost depends on the other |
+** constraints also present. As follows: |
+** |
+** * No other constraints: cost=1000.0 |
+** * One rowid range constraint: cost=750.0 |
+** * Both rowid range constraints: cost=500.0 |
+** * An == rowid constraint: cost=100.0 |
+** |
+** b) Otherwise, if there is no MATCH: |
+** |
+** * No other constraints: cost=1000000.0 |
+** * One rowid range constraint: cost=750000.0 |
+** * Both rowid range constraints: cost=250000.0 |
+** * An == rowid constraint: cost=10.0 |
+** |
+** Costs are not modified by the ORDER BY clause. |
+*/ |
+static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ |
+ Fts5Table *pTab = (Fts5Table*)pVTab; |
+ Fts5Config *pConfig = pTab->pConfig; |
+ int idxFlags = 0; /* Parameter passed through to xFilter() */ |
+ int bHasMatch; |
+ int iNext; |
+ int i; |
+ |
+ struct Constraint { |
+ int op; /* Mask against sqlite3_index_constraint.op */ |
+ int fts5op; /* FTS5 mask for idxFlags */ |
+ int iCol; /* 0==rowid, 1==tbl, 2==rank */ |
+ int omit; /* True to omit this if found */ |
+ int iConsIndex; /* Index in pInfo->aConstraint[] */ |
+ } aConstraint[] = { |
+ {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, |
+ FTS5_BI_MATCH, 1, 1, -1}, |
+ {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, |
+ FTS5_BI_RANK, 2, 1, -1}, |
+ {SQLITE_INDEX_CONSTRAINT_EQ, FTS5_BI_ROWID_EQ, 0, 0, -1}, |
+ {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE, |
+ FTS5_BI_ROWID_LE, 0, 0, -1}, |
+ {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, |
+ FTS5_BI_ROWID_GE, 0, 0, -1}, |
+ }; |
+ |
+ int aColMap[3]; |
+ aColMap[0] = -1; |
+ aColMap[1] = pConfig->nCol; |
+ aColMap[2] = pConfig->nCol+1; |
+ |
+ /* Set idxFlags flags for all WHERE clause terms that will be used. */ |
+ for(i=0; i<pInfo->nConstraint; i++){ |
+ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; |
+ int j; |
+ for(j=0; j<ArraySize(aConstraint); j++){ |
+ struct Constraint *pC = &aConstraint[j]; |
+ if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){ |
+ if( p->usable ){ |
+ pC->iConsIndex = i; |
+ idxFlags |= pC->fts5op; |
+ }else if( j==0 ){ |
+ /* As there exists an unusable MATCH constraint this is an |
+ ** unusable plan. Set a prohibitively high cost. */ |
+ pInfo->estimatedCost = 1e50; |
+ return SQLITE_OK; |
+ } |
+ } |
+ } |
+ } |
+ |
+ /* Set idxFlags flags for the ORDER BY clause */ |
+ if( pInfo->nOrderBy==1 ){ |
+ int iSort = pInfo->aOrderBy[0].iColumn; |
+ if( iSort==(pConfig->nCol+1) && BitFlagTest(idxFlags, FTS5_BI_MATCH) ){ |
+ idxFlags |= FTS5_BI_ORDER_RANK; |
+ }else if( iSort==-1 ){ |
+ idxFlags |= FTS5_BI_ORDER_ROWID; |
+ } |
+ if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){ |
+ pInfo->orderByConsumed = 1; |
+ if( pInfo->aOrderBy[0].desc ){ |
+ idxFlags |= FTS5_BI_ORDER_DESC; |
+ } |
+ } |
+ } |
+ |
+ /* Calculate the estimated cost based on the flags set in idxFlags. */ |
+ bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH); |
+ if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){ |
+ pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0; |
+ if( bHasMatch==0 ) fts5SetUniqueFlag(pInfo); |
+ }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ |
+ pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0; |
+ }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ |
+ pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0; |
+ }else{ |
+ pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0; |
+ } |
+ |
+ /* Assign argvIndex values to each constraint in use. */ |
+ iNext = 1; |
+ for(i=0; i<ArraySize(aConstraint); i++){ |
+ struct Constraint *pC = &aConstraint[i]; |
+ if( pC->iConsIndex>=0 ){ |
+ pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++; |
+ pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit; |
+ } |
+ } |
+ |
+ pInfo->idxNum = idxFlags; |
+ return SQLITE_OK; |
+} |
+ |
+static int fts5NewTransaction(Fts5Table *pTab){ |
+ Fts5Cursor *pCsr; |
+ for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ |
+ if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK; |
+ } |
+ return sqlite3Fts5StorageReset(pTab->pStorage); |
+} |
+ |
+/* |
+** Implementation of xOpen method. |
+*/ |
+static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ |
+ Fts5Table *pTab = (Fts5Table*)pVTab; |
+ Fts5Config *pConfig = pTab->pConfig; |
+ Fts5Cursor *pCsr = 0; /* New cursor object */ |
+ int nByte; /* Bytes of space to allocate */ |
+ int rc; /* Return code */ |
+ |
+ rc = fts5NewTransaction(pTab); |
+ if( rc==SQLITE_OK ){ |
+ nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); |
+ pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); |
+ if( pCsr ){ |
+ Fts5Global *pGlobal = pTab->pGlobal; |
+ memset(pCsr, 0, nByte); |
+ pCsr->aColumnSize = (int*)&pCsr[1]; |
+ pCsr->pNext = pGlobal->pCsr; |
+ pGlobal->pCsr = pCsr; |
+ pCsr->iCsrId = ++pGlobal->iNextId; |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ } |
+ } |
+ *ppCsr = (sqlite3_vtab_cursor*)pCsr; |
+ return rc; |
+} |
+ |
+static int fts5StmtType(Fts5Cursor *pCsr){ |
+ if( pCsr->ePlan==FTS5_PLAN_SCAN ){ |
+ return (pCsr->bDesc) ? FTS5_STMT_SCAN_DESC : FTS5_STMT_SCAN_ASC; |
+ } |
+ return FTS5_STMT_LOOKUP; |
+} |
+ |
+/* |
+** This function is called after the cursor passed as the only argument |
+** is moved to point at a different row. It clears all cached data |
+** specific to the previous row stored by the cursor object. |
+*/ |
+static void fts5CsrNewrow(Fts5Cursor *pCsr){ |
+ CsrFlagSet(pCsr, |
+ FTS5CSR_REQUIRE_CONTENT |
+ | FTS5CSR_REQUIRE_DOCSIZE |
+ | FTS5CSR_REQUIRE_INST |
+ | FTS5CSR_REQUIRE_POSLIST |
+ ); |
+} |
+ |
+static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ |
+ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); |
+ Fts5Auxdata *pData; |
+ Fts5Auxdata *pNext; |
+ |
+ sqlite3_free(pCsr->aInstIter); |
+ sqlite3_free(pCsr->aInst); |
+ if( pCsr->pStmt ){ |
+ int eStmt = fts5StmtType(pCsr); |
+ sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); |
+ } |
+ if( pCsr->pSorter ){ |
+ Fts5Sorter *pSorter = pCsr->pSorter; |
+ sqlite3_finalize(pSorter->pStmt); |
+ sqlite3_free(pSorter); |
+ } |
+ |
+ if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){ |
+ sqlite3Fts5ExprFree(pCsr->pExpr); |
+ } |
+ |
+ for(pData=pCsr->pAuxdata; pData; pData=pNext){ |
+ pNext = pData->pNext; |
+ if( pData->xDelete ) pData->xDelete(pData->pPtr); |
+ sqlite3_free(pData); |
+ } |
+ |
+ sqlite3_finalize(pCsr->pRankArgStmt); |
+ sqlite3_free(pCsr->apRankArg); |
+ |
+ if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ |
+ sqlite3_free(pCsr->zRank); |
+ sqlite3_free(pCsr->zRankArgs); |
+ } |
+ |
+ memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr)); |
+} |
+ |
+ |
+/* |
+** Close the cursor. For additional information see the documentation |
+** on the xClose method of the virtual table interface. |
+*/ |
+static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ |
+ if( pCursor ){ |
+ Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; |
+ Fts5Cursor **pp; |
+ |
+ fts5FreeCursorComponents(pCsr); |
+ /* Remove the cursor from the Fts5Global.pCsr list */ |
+ for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); |
+ *pp = pCsr->pNext; |
+ |
+ sqlite3_free(pCsr); |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+static int fts5SorterNext(Fts5Cursor *pCsr){ |
+ Fts5Sorter *pSorter = pCsr->pSorter; |
+ int rc; |
+ |
+ rc = sqlite3_step(pSorter->pStmt); |
+ if( rc==SQLITE_DONE ){ |
+ rc = SQLITE_OK; |
+ CsrFlagSet(pCsr, FTS5CSR_EOF); |
+ }else if( rc==SQLITE_ROW ){ |
+ const u8 *a; |
+ const u8 *aBlob; |
+ int nBlob; |
+ int i; |
+ int iOff = 0; |
+ rc = SQLITE_OK; |
+ |
+ pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); |
+ nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); |
+ aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); |
+ |
+ /* nBlob==0 in detail=none mode. */ |
+ if( nBlob>0 ){ |
+ for(i=0; i<(pSorter->nIdx-1); i++){ |
+ int iVal; |
+ a += fts5GetVarint32(a, iVal); |
+ iOff += iVal; |
+ pSorter->aIdx[i] = iOff; |
+ } |
+ pSorter->aIdx[i] = &aBlob[nBlob] - a; |
+ pSorter->aPoslist = a; |
+ } |
+ |
+ fts5CsrNewrow(pCsr); |
+ } |
+ |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Set the FTS5CSR_REQUIRE_RESEEK flag on all FTS5_PLAN_MATCH cursors |
+** open on table pTab. |
+*/ |
+static void fts5TripCursors(Fts5Table *pTab){ |
+ Fts5Cursor *pCsr; |
+ for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ |
+ if( pCsr->ePlan==FTS5_PLAN_MATCH |
+ && pCsr->base.pVtab==(sqlite3_vtab*)pTab |
+ ){ |
+ CsrFlagSet(pCsr, FTS5CSR_REQUIRE_RESEEK); |
+ } |
+ } |
+} |
+ |
+/* |
+** If the REQUIRE_RESEEK flag is set on the cursor passed as the first |
+** argument, close and reopen all Fts5IndexIter iterators that the cursor |
+** is using. Then attempt to move the cursor to a rowid equal to or laster |
+** (in the cursors sort order - ASC or DESC) than the current rowid. |
+** |
+** If the new rowid is not equal to the old, set output parameter *pbSkip |
+** to 1 before returning. Otherwise, leave it unchanged. |
+** |
+** Return SQLITE_OK if successful or if no reseek was required, or an |
+** error code if an error occurred. |
+*/ |
+static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ |
+ int rc = SQLITE_OK; |
+ assert( *pbSkip==0 ); |
+ if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_RESEEK) ){ |
+ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); |
+ int bDesc = pCsr->bDesc; |
+ i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); |
+ |
+ rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, iRowid, bDesc); |
+ if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){ |
+ *pbSkip = 1; |
+ } |
+ |
+ CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK); |
+ fts5CsrNewrow(pCsr); |
+ if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ |
+ CsrFlagSet(pCsr, FTS5CSR_EOF); |
+ *pbSkip = 1; |
+ } |
+ } |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Advance the cursor to the next row in the table that matches the |
+** search criteria. |
+** |
+** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned |
+** even if we reach end-of-file. The fts5EofMethod() will be called |
+** subsequently to determine whether or not an EOF was hit. |
+*/ |
+static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; |
+ int rc; |
+ |
+ assert( (pCsr->ePlan<3)== |
+ (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) |
+ ); |
+ assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) ); |
+ |
+ if( pCsr->ePlan<3 ){ |
+ int bSkip = 0; |
+ if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc; |
+ rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid); |
+ CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr)); |
+ fts5CsrNewrow(pCsr); |
+ }else{ |
+ switch( pCsr->ePlan ){ |
+ case FTS5_PLAN_SPECIAL: { |
+ CsrFlagSet(pCsr, FTS5CSR_EOF); |
+ rc = SQLITE_OK; |
+ break; |
+ } |
+ |
+ case FTS5_PLAN_SORTED_MATCH: { |
+ rc = fts5SorterNext(pCsr); |
+ break; |
+ } |
+ |
+ default: |
+ rc = sqlite3_step(pCsr->pStmt); |
+ if( rc!=SQLITE_ROW ){ |
+ CsrFlagSet(pCsr, FTS5CSR_EOF); |
+ rc = sqlite3_reset(pCsr->pStmt); |
+ }else{ |
+ rc = SQLITE_OK; |
+ } |
+ break; |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+ |
+static int fts5PrepareStatement( |
+ sqlite3_stmt **ppStmt, |
+ Fts5Config *pConfig, |
+ const char *zFmt, |
+ ... |
+){ |
+ sqlite3_stmt *pRet = 0; |
+ int rc; |
+ char *zSql; |
+ va_list ap; |
+ |
+ va_start(ap, zFmt); |
+ zSql = sqlite3_vmprintf(zFmt, ap); |
+ if( zSql==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0); |
+ if( rc!=SQLITE_OK ){ |
+ *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); |
+ } |
+ sqlite3_free(zSql); |
+ } |
+ |
+ va_end(ap); |
+ *ppStmt = pRet; |
+ return rc; |
+} |
+ |
+static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ |
+ Fts5Config *pConfig = pTab->pConfig; |
+ Fts5Sorter *pSorter; |
+ int nPhrase; |
+ int nByte; |
+ int rc; |
+ const char *zRank = pCsr->zRank; |
+ const char *zRankArgs = pCsr->zRankArgs; |
+ |
+ nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); |
+ nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); |
+ pSorter = (Fts5Sorter*)sqlite3_malloc(nByte); |
+ if( pSorter==0 ) return SQLITE_NOMEM; |
+ memset(pSorter, 0, nByte); |
+ pSorter->nIdx = nPhrase; |
+ |
+ /* TODO: It would be better to have some system for reusing statement |
+ ** handles here, rather than preparing a new one for each query. But that |
+ ** is not possible as SQLite reference counts the virtual table objects. |
+ ** And since the statement required here reads from this very virtual |
+ ** table, saving it creates a circular reference. |
+ ** |
+ ** If SQLite a built-in statement cache, this wouldn't be a problem. */ |
+ rc = fts5PrepareStatement(&pSorter->pStmt, pConfig, |
+ "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s", |
+ pConfig->zDb, pConfig->zName, zRank, pConfig->zName, |
+ (zRankArgs ? ", " : ""), |
+ (zRankArgs ? zRankArgs : ""), |
+ bDesc ? "DESC" : "ASC" |
+ ); |
+ |
+ pCsr->pSorter = pSorter; |
+ if( rc==SQLITE_OK ){ |
+ assert( pTab->pSortCsr==0 ); |
+ pTab->pSortCsr = pCsr; |
+ rc = fts5SorterNext(pCsr); |
+ pTab->pSortCsr = 0; |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ sqlite3_finalize(pSorter->pStmt); |
+ sqlite3_free(pSorter); |
+ pCsr->pSorter = 0; |
+ } |
+ |
+ return rc; |
+} |
+ |
+static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ |
+ int rc; |
+ Fts5Expr *pExpr = pCsr->pExpr; |
+ rc = sqlite3Fts5ExprFirst(pExpr, pTab->pIndex, pCsr->iFirstRowid, bDesc); |
+ if( sqlite3Fts5ExprEof(pExpr) ){ |
+ CsrFlagSet(pCsr, FTS5CSR_EOF); |
+ } |
+ fts5CsrNewrow(pCsr); |
+ return rc; |
+} |
+ |
+/* |
+** Process a "special" query. A special query is identified as one with a |
+** MATCH expression that begins with a '*' character. The remainder of |
+** the text passed to the MATCH operator are used as the special query |
+** parameters. |
+*/ |
+static int fts5SpecialMatch( |
+ Fts5Table *pTab, |
+ Fts5Cursor *pCsr, |
+ const char *zQuery |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ const char *z = zQuery; /* Special query text */ |
+ int n; /* Number of bytes in text at z */ |
+ |
+ while( z[0]==' ' ) z++; |
+ for(n=0; z[n] && z[n]!=' '; n++); |
+ |
+ assert( pTab->base.zErrMsg==0 ); |
+ pCsr->ePlan = FTS5_PLAN_SPECIAL; |
+ |
+ if( 0==sqlite3_strnicmp("reads", z, n) ){ |
+ pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->pIndex); |
+ } |
+ else if( 0==sqlite3_strnicmp("id", z, n) ){ |
+ pCsr->iSpecial = pCsr->iCsrId; |
+ } |
+ else{ |
+ /* An unrecognized directive. Return an error message. */ |
+ pTab->base.zErrMsg = sqlite3_mprintf("unknown special query: %.*s", n, z); |
+ rc = SQLITE_ERROR; |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Search for an auxiliary function named zName that can be used with table |
+** pTab. If one is found, return a pointer to the corresponding Fts5Auxiliary |
+** structure. Otherwise, if no such function exists, return NULL. |
+*/ |
+static Fts5Auxiliary *fts5FindAuxiliary(Fts5Table *pTab, const char *zName){ |
+ Fts5Auxiliary *pAux; |
+ |
+ for(pAux=pTab->pGlobal->pAux; pAux; pAux=pAux->pNext){ |
+ if( sqlite3_stricmp(zName, pAux->zFunc)==0 ) return pAux; |
+ } |
+ |
+ /* No function of the specified name was found. Return 0. */ |
+ return 0; |
+} |
+ |
+ |
+static int fts5FindRankFunction(Fts5Cursor *pCsr){ |
+ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); |
+ Fts5Config *pConfig = pTab->pConfig; |
+ int rc = SQLITE_OK; |
+ Fts5Auxiliary *pAux = 0; |
+ const char *zRank = pCsr->zRank; |
+ const char *zRankArgs = pCsr->zRankArgs; |
+ |
+ if( zRankArgs ){ |
+ char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs); |
+ if( zSql ){ |
+ sqlite3_stmt *pStmt = 0; |
+ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pStmt, 0); |
+ sqlite3_free(zSql); |
+ assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 ); |
+ if( rc==SQLITE_OK ){ |
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){ |
+ int nByte; |
+ pCsr->nRankArg = sqlite3_column_count(pStmt); |
+ nByte = sizeof(sqlite3_value*)*pCsr->nRankArg; |
+ pCsr->apRankArg = (sqlite3_value**)sqlite3Fts5MallocZero(&rc, nByte); |
+ if( rc==SQLITE_OK ){ |
+ int i; |
+ for(i=0; i<pCsr->nRankArg; i++){ |
+ pCsr->apRankArg[i] = sqlite3_column_value(pStmt, i); |
+ } |
+ } |
+ pCsr->pRankArgStmt = pStmt; |
+ }else{ |
+ rc = sqlite3_finalize(pStmt); |
+ assert( rc!=SQLITE_OK ); |
+ } |
+ } |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ pAux = fts5FindAuxiliary(pTab, zRank); |
+ if( pAux==0 ){ |
+ assert( pTab->base.zErrMsg==0 ); |
+ pTab->base.zErrMsg = sqlite3_mprintf("no such function: %s", zRank); |
+ rc = SQLITE_ERROR; |
+ } |
+ } |
+ |
+ pCsr->pRank = pAux; |
+ return rc; |
+} |
+ |
+ |
+static int fts5CursorParseRank( |
+ Fts5Config *pConfig, |
+ Fts5Cursor *pCsr, |
+ sqlite3_value *pRank |
+){ |
+ int rc = SQLITE_OK; |
+ if( pRank ){ |
+ const char *z = (const char*)sqlite3_value_text(pRank); |
+ char *zRank = 0; |
+ char *zRankArgs = 0; |
+ |
+ if( z==0 ){ |
+ if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR; |
+ }else{ |
+ rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ pCsr->zRank = zRank; |
+ pCsr->zRankArgs = zRankArgs; |
+ CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK); |
+ }else if( rc==SQLITE_ERROR ){ |
+ pCsr->base.pVtab->zErrMsg = sqlite3_mprintf( |
+ "parse error in rank function: %s", z |
+ ); |
+ } |
+ }else{ |
+ if( pConfig->zRank ){ |
+ pCsr->zRank = (char*)pConfig->zRank; |
+ pCsr->zRankArgs = (char*)pConfig->zRankArgs; |
+ }else{ |
+ pCsr->zRank = (char*)FTS5_DEFAULT_RANK; |
+ pCsr->zRankArgs = 0; |
+ } |
+ } |
+ return rc; |
+} |
+ |
+static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){ |
+ if( pVal ){ |
+ int eType = sqlite3_value_numeric_type(pVal); |
+ if( eType==SQLITE_INTEGER ){ |
+ return sqlite3_value_int64(pVal); |
+ } |
+ } |
+ return iDefault; |
+} |
+ |
+/* |
+** This is the xFilter interface for the virtual table. See |
+** the virtual table xFilter method documentation for additional |
+** information. |
+** |
+** There are three possible query strategies: |
+** |
+** 1. Full-text search using a MATCH operator. |
+** 2. A by-rowid lookup. |
+** 3. A full-table scan. |
+*/ |
+static int fts5FilterMethod( |
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ |
+ int idxNum, /* Strategy index */ |
+ const char *zUnused, /* Unused */ |
+ int nVal, /* Number of elements in apVal */ |
+ sqlite3_value **apVal /* Arguments for the indexing scheme */ |
+){ |
+ Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); |
+ Fts5Config *pConfig = pTab->pConfig; |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; |
+ int rc = SQLITE_OK; /* Error code */ |
+ int iVal = 0; /* Counter for apVal[] */ |
+ int bDesc; /* True if ORDER BY [rank|rowid] DESC */ |
+ int bOrderByRank; /* True if ORDER BY rank */ |
+ sqlite3_value *pMatch = 0; /* <tbl> MATCH ? expression (or NULL) */ |
+ sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ |
+ sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ |
+ sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ |
+ sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ |
+ char **pzErrmsg = pConfig->pzErrmsg; |
+ |
+ UNUSED_PARAM(zUnused); |
+ UNUSED_PARAM(nVal); |
+ |
+ if( pCsr->ePlan ){ |
+ fts5FreeCursorComponents(pCsr); |
+ memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); |
+ } |
+ |
+ assert( pCsr->pStmt==0 ); |
+ assert( pCsr->pExpr==0 ); |
+ assert( pCsr->csrflags==0 ); |
+ assert( pCsr->pRank==0 ); |
+ assert( pCsr->zRank==0 ); |
+ assert( pCsr->zRankArgs==0 ); |
+ |
+ assert( pzErrmsg==0 || pzErrmsg==&pTab->base.zErrMsg ); |
+ pConfig->pzErrmsg = &pTab->base.zErrMsg; |
+ |
+ /* Decode the arguments passed through to this function. |
+ ** |
+ ** Note: The following set of if(...) statements must be in the same |
+ ** order as the corresponding entries in the struct at the top of |
+ ** fts5BestIndexMethod(). */ |
+ if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++]; |
+ if( BitFlagTest(idxNum, FTS5_BI_RANK) ) pRank = apVal[iVal++]; |
+ if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++]; |
+ if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++]; |
+ if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++]; |
+ assert( iVal==nVal ); |
+ bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0); |
+ pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0); |
+ |
+ /* Set the cursor upper and lower rowid limits. Only some strategies |
+ ** actually use them. This is ok, as the xBestIndex() method leaves the |
+ ** sqlite3_index_constraint.omit flag clear for range constraints |
+ ** on the rowid field. */ |
+ if( pRowidEq ){ |
+ pRowidLe = pRowidGe = pRowidEq; |
+ } |
+ if( bDesc ){ |
+ pCsr->iFirstRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64); |
+ pCsr->iLastRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); |
+ }else{ |
+ pCsr->iLastRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64); |
+ pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); |
+ } |
+ |
+ if( pTab->pSortCsr ){ |
+ /* If pSortCsr is non-NULL, then this call is being made as part of |
+ ** processing for a "... MATCH <expr> ORDER BY rank" query (ePlan is |
+ ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will |
+ ** return results to the user for this query. The current cursor |
+ ** (pCursor) is used to execute the query issued by function |
+ ** fts5CursorFirstSorted() above. */ |
+ assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); |
+ assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); |
+ assert( pCsr->iLastRowid==LARGEST_INT64 ); |
+ assert( pCsr->iFirstRowid==SMALLEST_INT64 ); |
+ pCsr->ePlan = FTS5_PLAN_SOURCE; |
+ pCsr->pExpr = pTab->pSortCsr->pExpr; |
+ rc = fts5CursorFirst(pTab, pCsr, bDesc); |
+ }else if( pMatch ){ |
+ const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); |
+ if( zExpr==0 ) zExpr = ""; |
+ |
+ rc = fts5CursorParseRank(pConfig, pCsr, pRank); |
+ if( rc==SQLITE_OK ){ |
+ if( zExpr[0]=='*' ){ |
+ /* The user has issued a query of the form "MATCH '*...'". This |
+ ** indicates that the MATCH expression is not a full text query, |
+ ** but a request for an internal parameter. */ |
+ rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]); |
+ }else{ |
+ char **pzErr = &pTab->base.zErrMsg; |
+ rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr); |
+ if( rc==SQLITE_OK ){ |
+ if( bOrderByRank ){ |
+ pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; |
+ rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); |
+ }else{ |
+ pCsr->ePlan = FTS5_PLAN_MATCH; |
+ rc = fts5CursorFirst(pTab, pCsr, bDesc); |
+ } |
+ } |
+ } |
+ } |
+ }else if( pConfig->zContent==0 ){ |
+ *pConfig->pzErrmsg = sqlite3_mprintf( |
+ "%s: table does not support scanning", pConfig->zName |
+ ); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup |
+ ** by rowid (ePlan==FTS5_PLAN_ROWID). */ |
+ pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN); |
+ rc = sqlite3Fts5StorageStmt( |
+ pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->base.zErrMsg |
+ ); |
+ if( rc==SQLITE_OK ){ |
+ if( pCsr->ePlan==FTS5_PLAN_ROWID ){ |
+ sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); |
+ }else{ |
+ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid); |
+ sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid); |
+ } |
+ rc = fts5NextMethod(pCursor); |
+ } |
+ } |
+ |
+ pConfig->pzErrmsg = pzErrmsg; |
+ return rc; |
+} |
+ |
+/* |
+** This is the xEof method of the virtual table. SQLite calls this |
+** routine to find out if it has reached the end of a result set. |
+*/ |
+static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; |
+ return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0); |
+} |
+ |
+/* |
+** Return the rowid that the cursor currently points to. |
+*/ |
+static i64 fts5CursorRowid(Fts5Cursor *pCsr){ |
+ assert( pCsr->ePlan==FTS5_PLAN_MATCH |
+ || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH |
+ || pCsr->ePlan==FTS5_PLAN_SOURCE |
+ ); |
+ if( pCsr->pSorter ){ |
+ return pCsr->pSorter->iRowid; |
+ }else{ |
+ return sqlite3Fts5ExprRowid(pCsr->pExpr); |
+ } |
+} |
+ |
+/* |
+** This is the xRowid method. The SQLite core calls this routine to |
+** retrieve the rowid for the current row of the result set. fts5 |
+** exposes %_content.rowid as the rowid for the virtual table. The |
+** rowid should be written to *pRowid. |
+*/ |
+static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; |
+ int ePlan = pCsr->ePlan; |
+ |
+ assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); |
+ switch( ePlan ){ |
+ case FTS5_PLAN_SPECIAL: |
+ *pRowid = 0; |
+ break; |
+ |
+ case FTS5_PLAN_SOURCE: |
+ case FTS5_PLAN_MATCH: |
+ case FTS5_PLAN_SORTED_MATCH: |
+ *pRowid = fts5CursorRowid(pCsr); |
+ break; |
+ |
+ default: |
+ *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); |
+ break; |
+ } |
+ |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** If the cursor requires seeking (bSeekRequired flag is set), seek it. |
+** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise. |
+** |
+** If argument bErrormsg is true and an error occurs, an error message may |
+** be left in sqlite3_vtab.zErrMsg. |
+*/ |
+static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ |
+ int rc = SQLITE_OK; |
+ |
+ /* If the cursor does not yet have a statement handle, obtain one now. */ |
+ if( pCsr->pStmt==0 ){ |
+ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); |
+ int eStmt = fts5StmtType(pCsr); |
+ rc = sqlite3Fts5StorageStmt( |
+ pTab->pStorage, eStmt, &pCsr->pStmt, (bErrormsg?&pTab->base.zErrMsg:0) |
+ ); |
+ assert( rc!=SQLITE_OK || pTab->base.zErrMsg==0 ); |
+ assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ); |
+ } |
+ |
+ if( rc==SQLITE_OK && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){ |
+ assert( pCsr->pExpr ); |
+ sqlite3_reset(pCsr->pStmt); |
+ sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr)); |
+ rc = sqlite3_step(pCsr->pStmt); |
+ if( rc==SQLITE_ROW ){ |
+ rc = SQLITE_OK; |
+ CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT); |
+ }else{ |
+ rc = sqlite3_reset(pCsr->pStmt); |
+ if( rc==SQLITE_OK ){ |
+ rc = FTS5_CORRUPT; |
+ } |
+ } |
+ } |
+ return rc; |
+} |
+ |
+static void fts5SetVtabError(Fts5Table *p, const char *zFormat, ...){ |
+ va_list ap; /* ... printf arguments */ |
+ va_start(ap, zFormat); |
+ assert( p->base.zErrMsg==0 ); |
+ p->base.zErrMsg = sqlite3_vmprintf(zFormat, ap); |
+ va_end(ap); |
+} |
+ |
+/* |
+** This function is called to handle an FTS INSERT command. In other words, |
+** an INSERT statement of the form: |
+** |
+** INSERT INTO fts(fts) VALUES($pCmd) |
+** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal) |
+** |
+** Argument pVal is the value assigned to column "fts" by the INSERT |
+** statement. This function returns SQLITE_OK if successful, or an SQLite |
+** error code if an error occurs. |
+** |
+** The commands implemented by this function are documented in the "Special |
+** INSERT Directives" section of the documentation. It should be updated if |
+** more commands are added to this function. |
+*/ |
+static int fts5SpecialInsert( |
+ Fts5Table *pTab, /* Fts5 table object */ |
+ const char *zCmd, /* Text inserted into table-name column */ |
+ sqlite3_value *pVal /* Value inserted into rank column */ |
+){ |
+ Fts5Config *pConfig = pTab->pConfig; |
+ int rc = SQLITE_OK; |
+ int bError = 0; |
+ |
+ if( 0==sqlite3_stricmp("delete-all", zCmd) ){ |
+ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ |
+ fts5SetVtabError(pTab, |
+ "'delete-all' may only be used with a " |
+ "contentless or external content fts5 table" |
+ ); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage); |
+ } |
+ }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){ |
+ if( pConfig->eContent==FTS5_CONTENT_NONE ){ |
+ fts5SetVtabError(pTab, |
+ "'rebuild' may not be used with a contentless fts5 table" |
+ ); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ rc = sqlite3Fts5StorageRebuild(pTab->pStorage); |
+ } |
+ }else if( 0==sqlite3_stricmp("optimize", zCmd) ){ |
+ rc = sqlite3Fts5StorageOptimize(pTab->pStorage); |
+ }else if( 0==sqlite3_stricmp("merge", zCmd) ){ |
+ int nMerge = sqlite3_value_int(pVal); |
+ rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge); |
+ }else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){ |
+ rc = sqlite3Fts5StorageIntegrity(pTab->pStorage); |
+#ifdef SQLITE_DEBUG |
+ }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){ |
+ pConfig->bPrefixIndex = sqlite3_value_int(pVal); |
+#endif |
+ }else{ |
+ rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, zCmd, pVal, &bError); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ if( bError ){ |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, zCmd, pVal, 0); |
+ } |
+ } |
+ } |
+ return rc; |
+} |
+ |
+static int fts5SpecialDelete( |
+ Fts5Table *pTab, |
+ sqlite3_value **apVal |
+){ |
+ int rc = SQLITE_OK; |
+ int eType1 = sqlite3_value_type(apVal[1]); |
+ if( eType1==SQLITE_INTEGER ){ |
+ sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); |
+ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]); |
+ } |
+ return rc; |
+} |
+ |
+static void fts5StorageInsert( |
+ int *pRc, |
+ Fts5Table *pTab, |
+ sqlite3_value **apVal, |
+ i64 *piRowid |
+){ |
+ int rc = *pRc; |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, piRowid); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid); |
+ } |
+ *pRc = rc; |
+} |
+ |
+/* |
+** This function is the implementation of the xUpdate callback used by |
+** FTS3 virtual tables. It is invoked by SQLite each time a row is to be |
+** inserted, updated or deleted. |
+** |
+** A delete specifies a single argument - the rowid of the row to remove. |
+** |
+** Update and insert operations pass: |
+** |
+** 1. The "old" rowid, or NULL. |
+** 2. The "new" rowid. |
+** 3. Values for each of the nCol matchable columns. |
+** 4. Values for the two hidden columns (<tablename> and "rank"). |
+*/ |
+static int fts5UpdateMethod( |
+ sqlite3_vtab *pVtab, /* Virtual table handle */ |
+ int nArg, /* Size of argument array */ |
+ sqlite3_value **apVal, /* Array of arguments */ |
+ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ |
+){ |
+ Fts5Table *pTab = (Fts5Table*)pVtab; |
+ Fts5Config *pConfig = pTab->pConfig; |
+ int eType0; /* value_type() of apVal[0] */ |
+ int rc = SQLITE_OK; /* Return code */ |
+ |
+ /* A transaction must be open when this is called. */ |
+ assert( pTab->ts.eState==1 ); |
+ |
+ assert( pVtab->zErrMsg==0 ); |
+ assert( nArg==1 || nArg==(2+pConfig->nCol+2) ); |
+ assert( nArg==1 |
+ || sqlite3_value_type(apVal[1])==SQLITE_INTEGER |
+ || sqlite3_value_type(apVal[1])==SQLITE_NULL |
+ ); |
+ assert( pTab->pConfig->pzErrmsg==0 ); |
+ pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg; |
+ |
+ /* Put any active cursors into REQUIRE_SEEK state. */ |
+ fts5TripCursors(pTab); |
+ |
+ eType0 = sqlite3_value_type(apVal[0]); |
+ if( eType0==SQLITE_NULL |
+ && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL |
+ ){ |
+ /* A "special" INSERT op. These are handled separately. */ |
+ const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]); |
+ if( pConfig->eContent!=FTS5_CONTENT_NORMAL |
+ && 0==sqlite3_stricmp("delete", z) |
+ ){ |
+ rc = fts5SpecialDelete(pTab, apVal); |
+ }else{ |
+ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); |
+ } |
+ }else{ |
+ /* A regular INSERT, UPDATE or DELETE statement. The trick here is that |
+ ** any conflict on the rowid value must be detected before any |
+ ** modifications are made to the database file. There are 4 cases: |
+ ** |
+ ** 1) DELETE |
+ ** 2) UPDATE (rowid not modified) |
+ ** 3) UPDATE (rowid modified) |
+ ** 4) INSERT |
+ ** |
+ ** Cases 3 and 4 may violate the rowid constraint. |
+ */ |
+ int eConflict = SQLITE_ABORT; |
+ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ |
+ eConflict = sqlite3_vtab_on_conflict(pConfig->db); |
+ } |
+ |
+ assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); |
+ assert( nArg!=1 || eType0==SQLITE_INTEGER ); |
+ |
+ /* Filter out attempts to run UPDATE or DELETE on contentless tables. |
+ ** This is not suported. */ |
+ if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){ |
+ pTab->base.zErrMsg = sqlite3_mprintf( |
+ "cannot %s contentless fts5 table: %s", |
+ (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName |
+ ); |
+ rc = SQLITE_ERROR; |
+ } |
+ |
+ /* DELETE */ |
+ else if( nArg==1 ){ |
+ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ |
+ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); |
+ } |
+ |
+ /* INSERT */ |
+ else if( eType0!=SQLITE_INTEGER ){ |
+ /* If this is a REPLACE, first remove the current entry (if any) */ |
+ if( eConflict==SQLITE_REPLACE |
+ && sqlite3_value_type(apVal[1])==SQLITE_INTEGER |
+ ){ |
+ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ |
+ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); |
+ } |
+ fts5StorageInsert(&rc, pTab, apVal, pRowid); |
+ } |
+ |
+ /* UPDATE */ |
+ else{ |
+ i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ |
+ i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ |
+ if( iOld!=iNew ){ |
+ if( eConflict==SQLITE_REPLACE ){ |
+ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); |
+ } |
+ fts5StorageInsert(&rc, pTab, apVal, pRowid); |
+ }else{ |
+ rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid); |
+ } |
+ } |
+ }else{ |
+ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); |
+ fts5StorageInsert(&rc, pTab, apVal, pRowid); |
+ } |
+ } |
+ } |
+ |
+ pTab->pConfig->pzErrmsg = 0; |
+ return rc; |
+} |
+ |
+/* |
+** Implementation of xSync() method. |
+*/ |
+static int fts5SyncMethod(sqlite3_vtab *pVtab){ |
+ int rc; |
+ Fts5Table *pTab = (Fts5Table*)pVtab; |
+ fts5CheckTransactionState(pTab, FTS5_SYNC, 0); |
+ pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg; |
+ fts5TripCursors(pTab); |
+ rc = sqlite3Fts5StorageSync(pTab->pStorage, 1); |
+ pTab->pConfig->pzErrmsg = 0; |
+ return rc; |
+} |
+ |
+/* |
+** Implementation of xBegin() method. |
+*/ |
+static int fts5BeginMethod(sqlite3_vtab *pVtab){ |
+ fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0); |
+ fts5NewTransaction((Fts5Table*)pVtab); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Implementation of xCommit() method. This is a no-op. The contents of |
+** the pending-terms hash-table have already been flushed into the database |
+** by fts5SyncMethod(). |
+*/ |
+static int fts5CommitMethod(sqlite3_vtab *pVtab){ |
+ UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */ |
+ fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_COMMIT, 0); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Implementation of xRollback(). Discard the contents of the pending-terms |
+** hash-table. Any changes made to the database are reverted by SQLite. |
+*/ |
+static int fts5RollbackMethod(sqlite3_vtab *pVtab){ |
+ int rc; |
+ Fts5Table *pTab = (Fts5Table*)pVtab; |
+ fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); |
+ rc = sqlite3Fts5StorageRollback(pTab->pStorage); |
+ return rc; |
+} |
+ |
+static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*); |
+ |
+static void *fts5ApiUserData(Fts5Context *pCtx){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ return pCsr->pAux->pUserData; |
+} |
+ |
+static int fts5ApiColumnCount(Fts5Context *pCtx){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol; |
+} |
+ |
+static int fts5ApiColumnTotalSize( |
+ Fts5Context *pCtx, |
+ int iCol, |
+ sqlite3_int64 *pnToken |
+){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); |
+ return sqlite3Fts5StorageSize(pTab->pStorage, iCol, pnToken); |
+} |
+ |
+static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); |
+ return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow); |
+} |
+ |
+static int fts5ApiTokenize( |
+ Fts5Context *pCtx, |
+ const char *pText, int nText, |
+ void *pUserData, |
+ int (*xToken)(void*, int, const char*, int, int, int) |
+){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); |
+ return sqlite3Fts5Tokenize( |
+ pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken |
+ ); |
+} |
+ |
+static int fts5ApiPhraseCount(Fts5Context *pCtx){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ return sqlite3Fts5ExprPhraseCount(pCsr->pExpr); |
+} |
+ |
+static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); |
+} |
+ |
+static int fts5ApiColumnText( |
+ Fts5Context *pCtx, |
+ int iCol, |
+ const char **pz, |
+ int *pn |
+){ |
+ int rc = SQLITE_OK; |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ |
+ *pz = 0; |
+ *pn = 0; |
+ }else{ |
+ rc = fts5SeekCursor(pCsr, 0); |
+ if( rc==SQLITE_OK ){ |
+ *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); |
+ *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+static int fts5CsrPoslist( |
+ Fts5Cursor *pCsr, |
+ int iPhrase, |
+ const u8 **pa, |
+ int *pn |
+){ |
+ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; |
+ int rc = SQLITE_OK; |
+ int bLive = (pCsr->pSorter==0); |
+ |
+ if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ |
+ |
+ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ |
+ Fts5PoslistPopulator *aPopulator; |
+ int i; |
+ aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); |
+ if( aPopulator==0 ) rc = SQLITE_NOMEM; |
+ for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){ |
+ int n; const char *z; |
+ rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5ExprPopulatePoslists( |
+ pConfig, pCsr->pExpr, aPopulator, i, z, n |
+ ); |
+ } |
+ } |
+ sqlite3_free(aPopulator); |
+ |
+ if( pCsr->pSorter ){ |
+ sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); |
+ } |
+ } |
+ CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); |
+ } |
+ |
+ if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ |
+ Fts5Sorter *pSorter = pCsr->pSorter; |
+ int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); |
+ *pn = pSorter->aIdx[iPhrase] - i1; |
+ *pa = &pSorter->aPoslist[i1]; |
+ }else{ |
+ *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated |
+** correctly for the current view. Return SQLITE_OK if successful, or an |
+** SQLite error code otherwise. |
+*/ |
+static int fts5CacheInstArray(Fts5Cursor *pCsr){ |
+ int rc = SQLITE_OK; |
+ Fts5PoslistReader *aIter; /* One iterator for each phrase */ |
+ int nIter; /* Number of iterators/phrases */ |
+ |
+ nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); |
+ if( pCsr->aInstIter==0 ){ |
+ int nByte = sizeof(Fts5PoslistReader) * nIter; |
+ pCsr->aInstIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte); |
+ } |
+ aIter = pCsr->aInstIter; |
+ |
+ if( aIter ){ |
+ int nInst = 0; /* Number instances seen so far */ |
+ int i; |
+ |
+ /* Initialize all iterators */ |
+ for(i=0; i<nIter && rc==SQLITE_OK; i++){ |
+ const u8 *a; |
+ int n; |
+ rc = fts5CsrPoslist(pCsr, i, &a, &n); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ while( 1 ){ |
+ int *aInst; |
+ int iBest = -1; |
+ for(i=0; i<nIter; i++){ |
+ if( (aIter[i].bEof==0) |
+ && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) |
+ ){ |
+ iBest = i; |
+ } |
+ } |
+ if( iBest<0 ) break; |
+ |
+ nInst++; |
+ if( nInst>=pCsr->nInstAlloc ){ |
+ pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; |
+ aInst = (int*)sqlite3_realloc( |
+ pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 |
+ ); |
+ if( aInst ){ |
+ pCsr->aInst = aInst; |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ break; |
+ } |
+ } |
+ |
+ aInst = &pCsr->aInst[3 * (nInst-1)]; |
+ aInst[0] = iBest; |
+ aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); |
+ aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); |
+ sqlite3Fts5PoslistReaderNext(&aIter[iBest]); |
+ } |
+ } |
+ |
+ pCsr->nInstCount = nInst; |
+ CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST); |
+ } |
+ return rc; |
+} |
+ |
+static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ int rc = SQLITE_OK; |
+ if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 |
+ || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ |
+ *pnInst = pCsr->nInstCount; |
+ } |
+ return rc; |
+} |
+ |
+static int fts5ApiInst( |
+ Fts5Context *pCtx, |
+ int iIdx, |
+ int *piPhrase, |
+ int *piCol, |
+ int *piOff |
+){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ int rc = SQLITE_OK; |
+ if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 |
+ || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) |
+ ){ |
+ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ |
+ rc = SQLITE_RANGE; |
+#if 0 |
+ }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ |
+ *piPhrase = pCsr->aInst[iIdx*3]; |
+ *piCol = pCsr->aInst[iIdx*3 + 2]; |
+ *piOff = -1; |
+#endif |
+ }else{ |
+ *piPhrase = pCsr->aInst[iIdx*3]; |
+ *piCol = pCsr->aInst[iIdx*3 + 1]; |
+ *piOff = pCsr->aInst[iIdx*3 + 2]; |
+ } |
+ } |
+ return rc; |
+} |
+ |
+static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ |
+ return fts5CursorRowid((Fts5Cursor*)pCtx); |
+} |
+ |
+static int fts5ColumnSizeCb( |
+ void *pContext, /* Pointer to int */ |
+ int tflags, |
+ const char *pUnused, /* Buffer containing token */ |
+ int nUnused, /* Size of token in bytes */ |
+ int iUnused1, /* Start offset of token */ |
+ int iUnused2 /* End offset of token */ |
+){ |
+ int *pCnt = (int*)pContext; |
+ UNUSED_PARAM2(pUnused, nUnused); |
+ UNUSED_PARAM2(iUnused1, iUnused2); |
+ if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ |
+ (*pCnt)++; |
+ } |
+ return SQLITE_OK; |
+} |
+ |
+static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); |
+ Fts5Config *pConfig = pTab->pConfig; |
+ int rc = SQLITE_OK; |
+ |
+ if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){ |
+ if( pConfig->bColumnsize ){ |
+ i64 iRowid = fts5CursorRowid(pCsr); |
+ rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); |
+ }else if( pConfig->zContent==0 ){ |
+ int i; |
+ for(i=0; i<pConfig->nCol; i++){ |
+ if( pConfig->abUnindexed[i]==0 ){ |
+ pCsr->aColumnSize[i] = -1; |
+ } |
+ } |
+ }else{ |
+ int i; |
+ for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ |
+ if( pConfig->abUnindexed[i]==0 ){ |
+ const char *z; int n; |
+ void *p = (void*)(&pCsr->aColumnSize[i]); |
+ pCsr->aColumnSize[i] = 0; |
+ rc = fts5ApiColumnText(pCtx, i, &z, &n); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5Tokenize( |
+ pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb |
+ ); |
+ } |
+ } |
+ } |
+ } |
+ CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE); |
+ } |
+ if( iCol<0 ){ |
+ int i; |
+ *pnToken = 0; |
+ for(i=0; i<pConfig->nCol; i++){ |
+ *pnToken += pCsr->aColumnSize[i]; |
+ } |
+ }else if( iCol<pConfig->nCol ){ |
+ *pnToken = pCsr->aColumnSize[iCol]; |
+ }else{ |
+ *pnToken = 0; |
+ rc = SQLITE_RANGE; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Implementation of the xSetAuxdata() method. |
+*/ |
+static int fts5ApiSetAuxdata( |
+ Fts5Context *pCtx, /* Fts5 context */ |
+ void *pPtr, /* Pointer to save as auxdata */ |
+ void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */ |
+){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ Fts5Auxdata *pData; |
+ |
+ /* Search through the cursors list of Fts5Auxdata objects for one that |
+ ** corresponds to the currently executing auxiliary function. */ |
+ for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){ |
+ if( pData->pAux==pCsr->pAux ) break; |
+ } |
+ |
+ if( pData ){ |
+ if( pData->xDelete ){ |
+ pData->xDelete(pData->pPtr); |
+ } |
+ }else{ |
+ int rc = SQLITE_OK; |
+ pData = (Fts5Auxdata*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Auxdata)); |
+ if( pData==0 ){ |
+ if( xDelete ) xDelete(pPtr); |
+ return rc; |
+ } |
+ pData->pAux = pCsr->pAux; |
+ pData->pNext = pCsr->pAuxdata; |
+ pCsr->pAuxdata = pData; |
+ } |
+ |
+ pData->xDelete = xDelete; |
+ pData->pPtr = pPtr; |
+ return SQLITE_OK; |
+} |
+ |
+static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ Fts5Auxdata *pData; |
+ void *pRet = 0; |
+ |
+ for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){ |
+ if( pData->pAux==pCsr->pAux ) break; |
+ } |
+ |
+ if( pData ){ |
+ pRet = pData->pPtr; |
+ if( bClear ){ |
+ pData->pPtr = 0; |
+ pData->xDelete = 0; |
+ } |
+ } |
+ |
+ return pRet; |
+} |
+ |
+static void fts5ApiPhraseNext( |
+ Fts5Context *pUnused, |
+ Fts5PhraseIter *pIter, |
+ int *piCol, int *piOff |
+){ |
+ UNUSED_PARAM(pUnused); |
+ if( pIter->a>=pIter->b ){ |
+ *piCol = -1; |
+ *piOff = -1; |
+ }else{ |
+ int iVal; |
+ pIter->a += fts5GetVarint32(pIter->a, iVal); |
+ if( iVal==1 ){ |
+ pIter->a += fts5GetVarint32(pIter->a, iVal); |
+ *piCol = iVal; |
+ *piOff = 0; |
+ pIter->a += fts5GetVarint32(pIter->a, iVal); |
+ } |
+ *piOff += (iVal-2); |
+ } |
+} |
+ |
+static int fts5ApiPhraseFirst( |
+ Fts5Context *pCtx, |
+ int iPhrase, |
+ Fts5PhraseIter *pIter, |
+ int *piCol, int *piOff |
+){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ int n; |
+ int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); |
+ if( rc==SQLITE_OK ){ |
+ pIter->b = &pIter->a[n]; |
+ *piCol = 0; |
+ *piOff = 0; |
+ fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); |
+ } |
+ return rc; |
+} |
+ |
+static void fts5ApiPhraseNextColumn( |
+ Fts5Context *pCtx, |
+ Fts5PhraseIter *pIter, |
+ int *piCol |
+){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; |
+ |
+ if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ |
+ if( pIter->a>=pIter->b ){ |
+ *piCol = -1; |
+ }else{ |
+ int iIncr; |
+ pIter->a += fts5GetVarint32(&pIter->a[0], iIncr); |
+ *piCol += (iIncr-2); |
+ } |
+ }else{ |
+ while( 1 ){ |
+ int dummy; |
+ if( pIter->a>=pIter->b ){ |
+ *piCol = -1; |
+ return; |
+ } |
+ if( pIter->a[0]==0x01 ) break; |
+ pIter->a += fts5GetVarint32(pIter->a, dummy); |
+ } |
+ pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); |
+ } |
+} |
+ |
+static int fts5ApiPhraseFirstColumn( |
+ Fts5Context *pCtx, |
+ int iPhrase, |
+ Fts5PhraseIter *pIter, |
+ int *piCol |
+){ |
+ int rc = SQLITE_OK; |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; |
+ |
+ if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ |
+ Fts5Sorter *pSorter = pCsr->pSorter; |
+ int n; |
+ if( pSorter ){ |
+ int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); |
+ n = pSorter->aIdx[iPhrase] - i1; |
+ pIter->a = &pSorter->aPoslist[i1]; |
+ }else{ |
+ rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ pIter->b = &pIter->a[n]; |
+ *piCol = 0; |
+ fts5ApiPhraseNextColumn(pCtx, pIter, piCol); |
+ } |
+ }else{ |
+ int n; |
+ rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); |
+ if( rc==SQLITE_OK ){ |
+ pIter->b = &pIter->a[n]; |
+ if( n<=0 ){ |
+ *piCol = -1; |
+ }else if( pIter->a[0]==0x01 ){ |
+ pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); |
+ }else{ |
+ *piCol = 0; |
+ } |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+ |
+static int fts5ApiQueryPhrase(Fts5Context*, int, void*, |
+ int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) |
+); |
+ |
+static const Fts5ExtensionApi sFts5Api = { |
+ 2, /* iVersion */ |
+ fts5ApiUserData, |
+ fts5ApiColumnCount, |
+ fts5ApiRowCount, |
+ fts5ApiColumnTotalSize, |
+ fts5ApiTokenize, |
+ fts5ApiPhraseCount, |
+ fts5ApiPhraseSize, |
+ fts5ApiInstCount, |
+ fts5ApiInst, |
+ fts5ApiRowid, |
+ fts5ApiColumnText, |
+ fts5ApiColumnSize, |
+ fts5ApiQueryPhrase, |
+ fts5ApiSetAuxdata, |
+ fts5ApiGetAuxdata, |
+ fts5ApiPhraseFirst, |
+ fts5ApiPhraseNext, |
+ fts5ApiPhraseFirstColumn, |
+ fts5ApiPhraseNextColumn, |
+}; |
+ |
+/* |
+** Implementation of API function xQueryPhrase(). |
+*/ |
+static int fts5ApiQueryPhrase( |
+ Fts5Context *pCtx, |
+ int iPhrase, |
+ void *pUserData, |
+ int(*xCallback)(const Fts5ExtensionApi*, Fts5Context*, void*) |
+){ |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
+ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); |
+ int rc; |
+ Fts5Cursor *pNew = 0; |
+ |
+ rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew); |
+ if( rc==SQLITE_OK ){ |
+ pNew->ePlan = FTS5_PLAN_MATCH; |
+ pNew->iFirstRowid = SMALLEST_INT64; |
+ pNew->iLastRowid = LARGEST_INT64; |
+ pNew->base.pVtab = (sqlite3_vtab*)pTab; |
+ rc = sqlite3Fts5ExprClonePhrase(pCsr->pExpr, iPhrase, &pNew->pExpr); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ for(rc = fts5CursorFirst(pTab, pNew, 0); |
+ rc==SQLITE_OK && CsrFlagTest(pNew, FTS5CSR_EOF)==0; |
+ rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew) |
+ ){ |
+ rc = xCallback(&sFts5Api, (Fts5Context*)pNew, pUserData); |
+ if( rc!=SQLITE_OK ){ |
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK; |
+ break; |
+ } |
+ } |
+ } |
+ |
+ fts5CloseMethod((sqlite3_vtab_cursor*)pNew); |
+ return rc; |
+} |
+ |
+static void fts5ApiInvoke( |
+ Fts5Auxiliary *pAux, |
+ Fts5Cursor *pCsr, |
+ sqlite3_context *context, |
+ int argc, |
+ sqlite3_value **argv |
+){ |
+ assert( pCsr->pAux==0 ); |
+ pCsr->pAux = pAux; |
+ pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); |
+ pCsr->pAux = 0; |
+} |
+ |
+static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ |
+ Fts5Cursor *pCsr; |
+ for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ |
+ if( pCsr->iCsrId==iCsrId ) break; |
+ } |
+ return pCsr; |
+} |
+ |
+static void fts5ApiCallback( |
+ sqlite3_context *context, |
+ int argc, |
+ sqlite3_value **argv |
+){ |
+ |
+ Fts5Auxiliary *pAux; |
+ Fts5Cursor *pCsr; |
+ i64 iCsrId; |
+ |
+ assert( argc>=1 ); |
+ pAux = (Fts5Auxiliary*)sqlite3_user_data(context); |
+ iCsrId = sqlite3_value_int64(argv[0]); |
+ |
+ pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId); |
+ if( pCsr==0 ){ |
+ char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); |
+ sqlite3_result_error(context, zErr, -1); |
+ sqlite3_free(zErr); |
+ }else{ |
+ fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); |
+ } |
+} |
+ |
+ |
+/* |
+** Given cursor id iId, return a pointer to the corresponding Fts5Index |
+** object. Or NULL If the cursor id does not exist. |
+** |
+** If successful, set *ppConfig to point to the associated config object |
+** before returning. |
+*/ |
+static Fts5Index *sqlite3Fts5IndexFromCsrid( |
+ Fts5Global *pGlobal, /* FTS5 global context for db handle */ |
+ i64 iCsrId, /* Id of cursor to find */ |
+ Fts5Config **ppConfig /* OUT: Configuration object */ |
+){ |
+ Fts5Cursor *pCsr; |
+ Fts5Table *pTab; |
+ |
+ pCsr = fts5CursorFromCsrid(pGlobal, iCsrId); |
+ pTab = (Fts5Table*)pCsr->base.pVtab; |
+ *ppConfig = pTab->pConfig; |
+ |
+ return pTab->pIndex; |
+} |
+ |
+/* |
+** Return a "position-list blob" corresponding to the current position of |
+** cursor pCsr via sqlite3_result_blob(). A position-list blob contains |
+** the current position-list for each phrase in the query associated with |
+** cursor pCsr. |
+** |
+** A position-list blob begins with (nPhrase-1) varints, where nPhrase is |
+** the number of phrases in the query. Following the varints are the |
+** concatenated position lists for each phrase, in order. |
+** |
+** The first varint (if it exists) contains the size of the position list |
+** for phrase 0. The second (same disclaimer) contains the size of position |
+** list 1. And so on. There is no size field for the final position list, |
+** as it can be derived from the total size of the blob. |
+*/ |
+static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ |
+ int i; |
+ int rc = SQLITE_OK; |
+ int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); |
+ Fts5Buffer val; |
+ |
+ memset(&val, 0, sizeof(Fts5Buffer)); |
+ switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){ |
+ case FTS5_DETAIL_FULL: |
+ |
+ /* Append the varints */ |
+ for(i=0; i<(nPhrase-1); i++){ |
+ const u8 *dummy; |
+ int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); |
+ sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); |
+ } |
+ |
+ /* Append the position lists */ |
+ for(i=0; i<nPhrase; i++){ |
+ const u8 *pPoslist; |
+ int nPoslist; |
+ nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist); |
+ sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); |
+ } |
+ break; |
+ |
+ case FTS5_DETAIL_COLUMNS: |
+ |
+ /* Append the varints */ |
+ for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){ |
+ const u8 *dummy; |
+ int nByte; |
+ rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte); |
+ sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); |
+ } |
+ |
+ /* Append the position lists */ |
+ for(i=0; rc==SQLITE_OK && i<nPhrase; i++){ |
+ const u8 *pPoslist; |
+ int nPoslist; |
+ rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &pPoslist, &nPoslist); |
+ sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); |
+ } |
+ break; |
+ |
+ default: |
+ break; |
+ } |
+ |
+ sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); |
+ return rc; |
+} |
+ |
+/* |
+** This is the xColumn method, called by SQLite to request a value from |
+** the row that the supplied cursor currently points to. |
+*/ |
+static int fts5ColumnMethod( |
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ |
+ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ |
+ int iCol /* Index of column to read value from */ |
+){ |
+ Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); |
+ Fts5Config *pConfig = pTab->pConfig; |
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; |
+ int rc = SQLITE_OK; |
+ |
+ assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); |
+ |
+ if( pCsr->ePlan==FTS5_PLAN_SPECIAL ){ |
+ if( iCol==pConfig->nCol ){ |
+ sqlite3_result_int64(pCtx, pCsr->iSpecial); |
+ } |
+ }else |
+ |
+ if( iCol==pConfig->nCol ){ |
+ /* User is requesting the value of the special column with the same name |
+ ** as the table. Return the cursor integer id number. This value is only |
+ ** useful in that it may be passed as the first argument to an FTS5 |
+ ** auxiliary function. */ |
+ sqlite3_result_int64(pCtx, pCsr->iCsrId); |
+ }else if( iCol==pConfig->nCol+1 ){ |
+ |
+ /* The value of the "rank" column. */ |
+ if( pCsr->ePlan==FTS5_PLAN_SOURCE ){ |
+ fts5PoslistBlob(pCtx, pCsr); |
+ }else if( |
+ pCsr->ePlan==FTS5_PLAN_MATCH |
+ || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH |
+ ){ |
+ if( pCsr->pRank || SQLITE_OK==(rc = fts5FindRankFunction(pCsr)) ){ |
+ fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg); |
+ } |
+ } |
+ }else if( !fts5IsContentless(pTab) ){ |
+ rc = fts5SeekCursor(pCsr, 1); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+ |
+/* |
+** This routine implements the xFindFunction method for the FTS3 |
+** virtual table. |
+*/ |
+static int fts5FindFunctionMethod( |
+ sqlite3_vtab *pVtab, /* Virtual table handle */ |
+ int nUnused, /* Number of SQL function arguments */ |
+ const char *zName, /* Name of SQL function */ |
+ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ |
+ void **ppArg /* OUT: User data for *pxFunc */ |
+){ |
+ Fts5Table *pTab = (Fts5Table*)pVtab; |
+ Fts5Auxiliary *pAux; |
+ |
+ UNUSED_PARAM(nUnused); |
+ pAux = fts5FindAuxiliary(pTab, zName); |
+ if( pAux ){ |
+ *pxFunc = fts5ApiCallback; |
+ *ppArg = (void*)pAux; |
+ return 1; |
+ } |
+ |
+ /* No function of the specified name was found. Return 0. */ |
+ return 0; |
+} |
+ |
+/* |
+** Implementation of FTS5 xRename method. Rename an fts5 table. |
+*/ |
+static int fts5RenameMethod( |
+ sqlite3_vtab *pVtab, /* Virtual table handle */ |
+ const char *zName /* New name of table */ |
+){ |
+ Fts5Table *pTab = (Fts5Table*)pVtab; |
+ return sqlite3Fts5StorageRename(pTab->pStorage, zName); |
+} |
+ |
+/* |
+** The xSavepoint() method. |
+** |
+** Flush the contents of the pending-terms table to disk. |
+*/ |
+static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ |
+ Fts5Table *pTab = (Fts5Table*)pVtab; |
+ UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ |
+ fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint); |
+ fts5TripCursors(pTab); |
+ return sqlite3Fts5StorageSync(pTab->pStorage, 0); |
+} |
+ |
+/* |
+** The xRelease() method. |
+** |
+** This is a no-op. |
+*/ |
+static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ |
+ Fts5Table *pTab = (Fts5Table*)pVtab; |
+ UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ |
+ fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint); |
+ fts5TripCursors(pTab); |
+ return sqlite3Fts5StorageSync(pTab->pStorage, 0); |
+} |
+ |
+/* |
+** The xRollbackTo() method. |
+** |
+** Discard the contents of the pending terms table. |
+*/ |
+static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ |
+ Fts5Table *pTab = (Fts5Table*)pVtab; |
+ UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ |
+ fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); |
+ fts5TripCursors(pTab); |
+ return sqlite3Fts5StorageRollback(pTab->pStorage); |
+} |
+ |
+/* |
+** Register a new auxiliary function with global context pGlobal. |
+*/ |
+static int fts5CreateAux( |
+ fts5_api *pApi, /* Global context (one per db handle) */ |
+ const char *zName, /* Name of new function */ |
+ void *pUserData, /* User data for aux. function */ |
+ fts5_extension_function xFunc, /* Aux. function implementation */ |
+ void(*xDestroy)(void*) /* Destructor for pUserData */ |
+){ |
+ Fts5Global *pGlobal = (Fts5Global*)pApi; |
+ int rc = sqlite3_overload_function(pGlobal->db, zName, -1); |
+ if( rc==SQLITE_OK ){ |
+ Fts5Auxiliary *pAux; |
+ int nName; /* Size of zName in bytes, including \0 */ |
+ int nByte; /* Bytes of space to allocate */ |
+ |
+ nName = (int)strlen(zName) + 1; |
+ nByte = sizeof(Fts5Auxiliary) + nName; |
+ pAux = (Fts5Auxiliary*)sqlite3_malloc(nByte); |
+ if( pAux ){ |
+ memset(pAux, 0, nByte); |
+ pAux->zFunc = (char*)&pAux[1]; |
+ memcpy(pAux->zFunc, zName, nName); |
+ pAux->pGlobal = pGlobal; |
+ pAux->pUserData = pUserData; |
+ pAux->xFunc = xFunc; |
+ pAux->xDestroy = xDestroy; |
+ pAux->pNext = pGlobal->pAux; |
+ pGlobal->pAux = pAux; |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Register a new tokenizer. This is the implementation of the |
+** fts5_api.xCreateTokenizer() method. |
+*/ |
+static int fts5CreateTokenizer( |
+ fts5_api *pApi, /* Global context (one per db handle) */ |
+ const char *zName, /* Name of new function */ |
+ void *pUserData, /* User data for aux. function */ |
+ fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ |
+ void(*xDestroy)(void*) /* Destructor for pUserData */ |
+){ |
+ Fts5Global *pGlobal = (Fts5Global*)pApi; |
+ Fts5TokenizerModule *pNew; |
+ int nName; /* Size of zName and its \0 terminator */ |
+ int nByte; /* Bytes of space to allocate */ |
+ int rc = SQLITE_OK; |
+ |
+ nName = (int)strlen(zName) + 1; |
+ nByte = sizeof(Fts5TokenizerModule) + nName; |
+ pNew = (Fts5TokenizerModule*)sqlite3_malloc(nByte); |
+ if( pNew ){ |
+ memset(pNew, 0, nByte); |
+ pNew->zName = (char*)&pNew[1]; |
+ memcpy(pNew->zName, zName, nName); |
+ pNew->pUserData = pUserData; |
+ pNew->x = *pTokenizer; |
+ pNew->xDestroy = xDestroy; |
+ pNew->pNext = pGlobal->pTok; |
+ pGlobal->pTok = pNew; |
+ if( pNew->pNext==0 ){ |
+ pGlobal->pDfltTok = pNew; |
+ } |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ } |
+ |
+ return rc; |
+} |
+ |
+static Fts5TokenizerModule *fts5LocateTokenizer( |
+ Fts5Global *pGlobal, |
+ const char *zName |
+){ |
+ Fts5TokenizerModule *pMod = 0; |
+ |
+ if( zName==0 ){ |
+ pMod = pGlobal->pDfltTok; |
+ }else{ |
+ for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){ |
+ if( sqlite3_stricmp(zName, pMod->zName)==0 ) break; |
+ } |
+ } |
+ |
+ return pMod; |
+} |
+ |
+/* |
+** Find a tokenizer. This is the implementation of the |
+** fts5_api.xFindTokenizer() method. |
+*/ |
+static int fts5FindTokenizer( |
+ fts5_api *pApi, /* Global context (one per db handle) */ |
+ const char *zName, /* Name of new function */ |
+ void **ppUserData, |
+ fts5_tokenizer *pTokenizer /* Populate this object */ |
+){ |
+ int rc = SQLITE_OK; |
+ Fts5TokenizerModule *pMod; |
+ |
+ pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); |
+ if( pMod ){ |
+ *pTokenizer = pMod->x; |
+ *ppUserData = pMod->pUserData; |
+ }else{ |
+ memset(pTokenizer, 0, sizeof(fts5_tokenizer)); |
+ rc = SQLITE_ERROR; |
+ } |
+ |
+ return rc; |
+} |
+ |
+static int sqlite3Fts5GetTokenizer( |
+ Fts5Global *pGlobal, |
+ const char **azArg, |
+ int nArg, |
+ Fts5Tokenizer **ppTok, |
+ fts5_tokenizer **ppTokApi, |
+ char **pzErr |
+){ |
+ Fts5TokenizerModule *pMod; |
+ int rc = SQLITE_OK; |
+ |
+ pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]); |
+ if( pMod==0 ){ |
+ assert( nArg>0 ); |
+ rc = SQLITE_ERROR; |
+ *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); |
+ }else{ |
+ rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok); |
+ *ppTokApi = &pMod->x; |
+ if( rc!=SQLITE_OK && pzErr ){ |
+ *pzErr = sqlite3_mprintf("error in tokenizer constructor"); |
+ } |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ *ppTokApi = 0; |
+ *ppTok = 0; |
+ } |
+ |
+ return rc; |
+} |
+ |
+static void fts5ModuleDestroy(void *pCtx){ |
+ Fts5TokenizerModule *pTok, *pNextTok; |
+ Fts5Auxiliary *pAux, *pNextAux; |
+ Fts5Global *pGlobal = (Fts5Global*)pCtx; |
+ |
+ for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){ |
+ pNextAux = pAux->pNext; |
+ if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData); |
+ sqlite3_free(pAux); |
+ } |
+ |
+ for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){ |
+ pNextTok = pTok->pNext; |
+ if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData); |
+ sqlite3_free(pTok); |
+ } |
+ |
+ sqlite3_free(pGlobal); |
+} |
+ |
+static void fts5Fts5Func( |
+ sqlite3_context *pCtx, /* Function call context */ |
+ int nArg, /* Number of args */ |
+ sqlite3_value **apUnused /* Function arguments */ |
+){ |
+ Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); |
+ char buf[8]; |
+ UNUSED_PARAM2(nArg, apUnused); |
+ assert( nArg==0 ); |
+ assert( sizeof(buf)>=sizeof(pGlobal) ); |
+ memcpy(buf, (void*)&pGlobal, sizeof(pGlobal)); |
+ sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT); |
+} |
+ |
+/* |
+** Implementation of fts5_source_id() function. |
+*/ |
+static void fts5SourceIdFunc( |
+ sqlite3_context *pCtx, /* Function call context */ |
+ int nArg, /* Number of args */ |
+ sqlite3_value **apUnused /* Function arguments */ |
+){ |
+ assert( nArg==0 ); |
+ UNUSED_PARAM2(nArg, apUnused); |
+ sqlite3_result_text(pCtx, "fts5: 2017-02-13 16:02:40 ada05cfa86ad7f5645450ac7a2a21c9aa6e57d2c", -1, SQLITE_TRANSIENT); |
+} |
+ |
+static int fts5Init(sqlite3 *db){ |
+ static const sqlite3_module fts5Mod = { |
+ /* iVersion */ 2, |
+ /* xCreate */ fts5CreateMethod, |
+ /* xConnect */ fts5ConnectMethod, |
+ /* xBestIndex */ fts5BestIndexMethod, |
+ /* xDisconnect */ fts5DisconnectMethod, |
+ /* xDestroy */ fts5DestroyMethod, |
+ /* xOpen */ fts5OpenMethod, |
+ /* xClose */ fts5CloseMethod, |
+ /* xFilter */ fts5FilterMethod, |
+ /* xNext */ fts5NextMethod, |
+ /* xEof */ fts5EofMethod, |
+ /* xColumn */ fts5ColumnMethod, |
+ /* xRowid */ fts5RowidMethod, |
+ /* xUpdate */ fts5UpdateMethod, |
+ /* xBegin */ fts5BeginMethod, |
+ /* xSync */ fts5SyncMethod, |
+ /* xCommit */ fts5CommitMethod, |
+ /* xRollback */ fts5RollbackMethod, |
+ /* xFindFunction */ fts5FindFunctionMethod, |
+ /* xRename */ fts5RenameMethod, |
+ /* xSavepoint */ fts5SavepointMethod, |
+ /* xRelease */ fts5ReleaseMethod, |
+ /* xRollbackTo */ fts5RollbackToMethod, |
+ }; |
+ |
+ int rc; |
+ Fts5Global *pGlobal = 0; |
+ |
+ pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global)); |
+ if( pGlobal==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ void *p = (void*)pGlobal; |
+ memset(pGlobal, 0, sizeof(Fts5Global)); |
+ pGlobal->db = db; |
+ pGlobal->api.iVersion = 2; |
+ pGlobal->api.xCreateFunction = fts5CreateAux; |
+ pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; |
+ pGlobal->api.xFindTokenizer = fts5FindTokenizer; |
+ rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); |
+ if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); |
+ if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); |
+ if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api); |
+ if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api); |
+ if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db); |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3_create_function( |
+ db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 |
+ ); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3_create_function( |
+ db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0 |
+ ); |
+ } |
+ } |
+ |
+ /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file |
+ ** fts5_test_mi.c is compiled and linked into the executable. And call |
+ ** its entry point to enable the matchinfo() demo. */ |
+#ifdef SQLITE_FTS5_ENABLE_TEST_MI |
+ if( rc==SQLITE_OK ){ |
+ extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); |
+ rc = sqlite3Fts5TestRegisterMatchinfo(db); |
+ } |
+#endif |
+ |
+ return rc; |
+} |
+ |
+/* |
+** The following functions are used to register the module with SQLite. If |
+** this module is being built as part of the SQLite core (SQLITE_CORE is |
+** defined), then sqlite3_open() will call sqlite3Fts5Init() directly. |
+** |
+** Or, if this module is being built as a loadable extension, |
+** sqlite3Fts5Init() is omitted and the two standard entry points |
+** sqlite3_fts_init() and sqlite3_fts5_init() defined instead. |
+*/ |
+#ifndef SQLITE_CORE |
+#ifdef _WIN32 |
+__declspec(dllexport) |
+#endif |
+SQLITE_API int sqlite3_fts_init( |
+ sqlite3 *db, |
+ char **pzErrMsg, |
+ const sqlite3_api_routines *pApi |
+){ |
+ SQLITE_EXTENSION_INIT2(pApi); |
+ (void)pzErrMsg; /* Unused parameter */ |
+ return fts5Init(db); |
+} |
+ |
+#ifdef _WIN32 |
+__declspec(dllexport) |
+#endif |
+SQLITE_API int sqlite3_fts5_init( |
+ sqlite3 *db, |
+ char **pzErrMsg, |
+ const sqlite3_api_routines *pApi |
+){ |
+ SQLITE_EXTENSION_INIT2(pApi); |
+ (void)pzErrMsg; /* Unused parameter */ |
+ return fts5Init(db); |
+} |
+#else |
+SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3 *db){ |
+ return fts5Init(db); |
+} |
+#endif |
+ |
+/* |
+** 2014 May 31 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+*/ |
+ |
+ |
+ |
+/* #include "fts5Int.h" */ |
+ |
+struct Fts5Storage { |
+ Fts5Config *pConfig; |
+ Fts5Index *pIndex; |
+ int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */ |
+ i64 nTotalRow; /* Total number of rows in FTS table */ |
+ i64 *aTotalSize; /* Total sizes of each column */ |
+ sqlite3_stmt *aStmt[11]; |
+}; |
+ |
+ |
+#if FTS5_STMT_SCAN_ASC!=0 |
+# error "FTS5_STMT_SCAN_ASC mismatch" |
+#endif |
+#if FTS5_STMT_SCAN_DESC!=1 |
+# error "FTS5_STMT_SCAN_DESC mismatch" |
+#endif |
+#if FTS5_STMT_LOOKUP!=2 |
+# error "FTS5_STMT_LOOKUP mismatch" |
+#endif |
+ |
+#define FTS5_STMT_INSERT_CONTENT 3 |
+#define FTS5_STMT_REPLACE_CONTENT 4 |
+#define FTS5_STMT_DELETE_CONTENT 5 |
+#define FTS5_STMT_REPLACE_DOCSIZE 6 |
+#define FTS5_STMT_DELETE_DOCSIZE 7 |
+#define FTS5_STMT_LOOKUP_DOCSIZE 8 |
+#define FTS5_STMT_REPLACE_CONFIG 9 |
+#define FTS5_STMT_SCAN 10 |
+ |
+/* |
+** Prepare the two insert statements - Fts5Storage.pInsertContent and |
+** Fts5Storage.pInsertDocsize - if they have not already been prepared. |
+** Return SQLITE_OK if successful, or an SQLite error code if an error |
+** occurs. |
+*/ |
+static int fts5StorageGetStmt( |
+ Fts5Storage *p, /* Storage handle */ |
+ int eStmt, /* FTS5_STMT_XXX constant */ |
+ sqlite3_stmt **ppStmt, /* OUT: Prepared statement handle */ |
+ char **pzErrMsg /* OUT: Error message (if any) */ |
+){ |
+ int rc = SQLITE_OK; |
+ |
+ /* If there is no %_docsize table, there should be no requests for |
+ ** statements to operate on it. */ |
+ assert( p->pConfig->bColumnsize || ( |
+ eStmt!=FTS5_STMT_REPLACE_DOCSIZE |
+ && eStmt!=FTS5_STMT_DELETE_DOCSIZE |
+ && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE |
+ )); |
+ |
+ assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) ); |
+ if( p->aStmt[eStmt]==0 ){ |
+ const char *azStmt[] = { |
+ "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC", |
+ "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC", |
+ "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */ |
+ |
+ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ |
+ "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ |
+ "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ |
+ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */ |
+ "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ |
+ |
+ "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ |
+ |
+ "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */ |
+ "SELECT %s FROM %s AS T", /* SCAN */ |
+ }; |
+ Fts5Config *pC = p->pConfig; |
+ char *zSql = 0; |
+ |
+ switch( eStmt ){ |
+ case FTS5_STMT_SCAN: |
+ zSql = sqlite3_mprintf(azStmt[eStmt], |
+ pC->zContentExprlist, pC->zContent |
+ ); |
+ break; |
+ |
+ case FTS5_STMT_SCAN_ASC: |
+ case FTS5_STMT_SCAN_DESC: |
+ zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, |
+ pC->zContent, pC->zContentRowid, pC->zContentRowid, |
+ pC->zContentRowid |
+ ); |
+ break; |
+ |
+ case FTS5_STMT_LOOKUP: |
+ zSql = sqlite3_mprintf(azStmt[eStmt], |
+ pC->zContentExprlist, pC->zContent, pC->zContentRowid |
+ ); |
+ break; |
+ |
+ case FTS5_STMT_INSERT_CONTENT: |
+ case FTS5_STMT_REPLACE_CONTENT: { |
+ int nCol = pC->nCol + 1; |
+ char *zBind; |
+ int i; |
+ |
+ zBind = sqlite3_malloc(1 + nCol*2); |
+ if( zBind ){ |
+ for(i=0; i<nCol; i++){ |
+ zBind[i*2] = '?'; |
+ zBind[i*2 + 1] = ','; |
+ } |
+ zBind[i*2-1] = '\0'; |
+ zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind); |
+ sqlite3_free(zBind); |
+ } |
+ break; |
+ } |
+ |
+ default: |
+ zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); |
+ break; |
+ } |
+ |
+ if( zSql==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0); |
+ sqlite3_free(zSql); |
+ if( rc!=SQLITE_OK && pzErrMsg ){ |
+ *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); |
+ } |
+ } |
+ } |
+ |
+ *ppStmt = p->aStmt[eStmt]; |
+ sqlite3_reset(*ppStmt); |
+ return rc; |
+} |
+ |
+ |
+static int fts5ExecPrintf( |
+ sqlite3 *db, |
+ char **pzErr, |
+ const char *zFormat, |
+ ... |
+){ |
+ int rc; |
+ va_list ap; /* ... printf arguments */ |
+ char *zSql; |
+ |
+ va_start(ap, zFormat); |
+ zSql = sqlite3_vmprintf(zFormat, ap); |
+ |
+ if( zSql==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ rc = sqlite3_exec(db, zSql, 0, 0, pzErr); |
+ sqlite3_free(zSql); |
+ } |
+ |
+ va_end(ap); |
+ return rc; |
+} |
+ |
+/* |
+** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error |
+** code otherwise. |
+*/ |
+static int sqlite3Fts5DropAll(Fts5Config *pConfig){ |
+ int rc = fts5ExecPrintf(pConfig->db, 0, |
+ "DROP TABLE IF EXISTS %Q.'%q_data';" |
+ "DROP TABLE IF EXISTS %Q.'%q_idx';" |
+ "DROP TABLE IF EXISTS %Q.'%q_config';", |
+ pConfig->zDb, pConfig->zName, |
+ pConfig->zDb, pConfig->zName, |
+ pConfig->zDb, pConfig->zName |
+ ); |
+ if( rc==SQLITE_OK && pConfig->bColumnsize ){ |
+ rc = fts5ExecPrintf(pConfig->db, 0, |
+ "DROP TABLE IF EXISTS %Q.'%q_docsize';", |
+ pConfig->zDb, pConfig->zName |
+ ); |
+ } |
+ if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ |
+ rc = fts5ExecPrintf(pConfig->db, 0, |
+ "DROP TABLE IF EXISTS %Q.'%q_content';", |
+ pConfig->zDb, pConfig->zName |
+ ); |
+ } |
+ return rc; |
+} |
+ |
+static void fts5StorageRenameOne( |
+ Fts5Config *pConfig, /* Current FTS5 configuration */ |
+ int *pRc, /* IN/OUT: Error code */ |
+ const char *zTail, /* Tail of table name e.g. "data", "config" */ |
+ const char *zName /* New name of FTS5 table */ |
+){ |
+ if( *pRc==SQLITE_OK ){ |
+ *pRc = fts5ExecPrintf(pConfig->db, 0, |
+ "ALTER TABLE %Q.'%q_%s' RENAME TO '%q_%s';", |
+ pConfig->zDb, pConfig->zName, zTail, zName, zTail |
+ ); |
+ } |
+} |
+ |
+static int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){ |
+ Fts5Config *pConfig = pStorage->pConfig; |
+ int rc = sqlite3Fts5StorageSync(pStorage, 1); |
+ |
+ fts5StorageRenameOne(pConfig, &rc, "data", zName); |
+ fts5StorageRenameOne(pConfig, &rc, "idx", zName); |
+ fts5StorageRenameOne(pConfig, &rc, "config", zName); |
+ if( pConfig->bColumnsize ){ |
+ fts5StorageRenameOne(pConfig, &rc, "docsize", zName); |
+ } |
+ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ |
+ fts5StorageRenameOne(pConfig, &rc, "content", zName); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Create the shadow table named zPost, with definition zDefn. Return |
+** SQLITE_OK if successful, or an SQLite error code otherwise. |
+*/ |
+static int sqlite3Fts5CreateTable( |
+ Fts5Config *pConfig, /* FTS5 configuration */ |
+ const char *zPost, /* Shadow table to create (e.g. "content") */ |
+ const char *zDefn, /* Columns etc. for shadow table */ |
+ int bWithout, /* True for without rowid */ |
+ char **pzErr /* OUT: Error message */ |
+){ |
+ int rc; |
+ char *zErr = 0; |
+ |
+ rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s", |
+ pConfig->zDb, pConfig->zName, zPost, zDefn, |
+#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID |
+ bWithout?" WITHOUT ROWID": |
+#endif |
+ "" |
+ ); |
+ if( zErr ){ |
+ *pzErr = sqlite3_mprintf( |
+ "fts5: error creating shadow table %q_%s: %s", |
+ pConfig->zName, zPost, zErr |
+ ); |
+ sqlite3_free(zErr); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Open a new Fts5Index handle. If the bCreate argument is true, create |
+** and initialize the underlying tables |
+** |
+** If successful, set *pp to point to the new object and return SQLITE_OK. |
+** Otherwise, set *pp to NULL and return an SQLite error code. |
+*/ |
+static int sqlite3Fts5StorageOpen( |
+ Fts5Config *pConfig, |
+ Fts5Index *pIndex, |
+ int bCreate, |
+ Fts5Storage **pp, |
+ char **pzErr /* OUT: Error message */ |
+){ |
+ int rc = SQLITE_OK; |
+ Fts5Storage *p; /* New object */ |
+ int nByte; /* Bytes of space to allocate */ |
+ |
+ nByte = sizeof(Fts5Storage) /* Fts5Storage object */ |
+ + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */ |
+ *pp = p = (Fts5Storage*)sqlite3_malloc(nByte); |
+ if( !p ) return SQLITE_NOMEM; |
+ |
+ memset(p, 0, nByte); |
+ p->aTotalSize = (i64*)&p[1]; |
+ p->pConfig = pConfig; |
+ p->pIndex = pIndex; |
+ |
+ if( bCreate ){ |
+ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ |
+ int nDefn = 32 + pConfig->nCol*10; |
+ char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10); |
+ if( zDefn==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ int i; |
+ int iOff; |
+ sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY"); |
+ iOff = (int)strlen(zDefn); |
+ for(i=0; i<pConfig->nCol; i++){ |
+ sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); |
+ iOff += (int)strlen(&zDefn[iOff]); |
+ } |
+ rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); |
+ } |
+ sqlite3_free(zDefn); |
+ } |
+ |
+ if( rc==SQLITE_OK && pConfig->bColumnsize ){ |
+ rc = sqlite3Fts5CreateTable( |
+ pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr |
+ ); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5CreateTable( |
+ pConfig, "config", "k PRIMARY KEY, v", 1, pzErr |
+ ); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); |
+ } |
+ } |
+ |
+ if( rc ){ |
+ sqlite3Fts5StorageClose(p); |
+ *pp = 0; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen(). |
+*/ |
+static int sqlite3Fts5StorageClose(Fts5Storage *p){ |
+ int rc = SQLITE_OK; |
+ if( p ){ |
+ int i; |
+ |
+ /* Finalize all SQL statements */ |
+ for(i=0; i<ArraySize(p->aStmt); i++){ |
+ sqlite3_finalize(p->aStmt[i]); |
+ } |
+ |
+ sqlite3_free(p); |
+ } |
+ return rc; |
+} |
+ |
+typedef struct Fts5InsertCtx Fts5InsertCtx; |
+struct Fts5InsertCtx { |
+ Fts5Storage *pStorage; |
+ int iCol; |
+ int szCol; /* Size of column value in tokens */ |
+}; |
+ |
+/* |
+** Tokenization callback used when inserting tokens into the FTS index. |
+*/ |
+static int fts5StorageInsertCallback( |
+ void *pContext, /* Pointer to Fts5InsertCtx object */ |
+ int tflags, |
+ const char *pToken, /* Buffer containing token */ |
+ int nToken, /* Size of token in bytes */ |
+ int iUnused1, /* Start offset of token */ |
+ int iUnused2 /* End offset of token */ |
+){ |
+ Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; |
+ Fts5Index *pIdx = pCtx->pStorage->pIndex; |
+ UNUSED_PARAM2(iUnused1, iUnused2); |
+ if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; |
+ if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ |
+ pCtx->szCol++; |
+ } |
+ return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); |
+} |
+ |
+/* |
+** If a row with rowid iDel is present in the %_content table, add the |
+** delete-markers to the FTS index necessary to delete it. Do not actually |
+** remove the %_content row at this time though. |
+*/ |
+static int fts5StorageDeleteFromIndex( |
+ Fts5Storage *p, |
+ i64 iDel, |
+ sqlite3_value **apVal |
+){ |
+ Fts5Config *pConfig = p->pConfig; |
+ sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ |
+ int rc; /* Return code */ |
+ int rc2; /* sqlite3_reset() return code */ |
+ int iCol; |
+ Fts5InsertCtx ctx; |
+ |
+ if( apVal==0 ){ |
+ rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); |
+ if( rc!=SQLITE_OK ) return rc; |
+ sqlite3_bind_int64(pSeek, 1, iDel); |
+ if( sqlite3_step(pSeek)!=SQLITE_ROW ){ |
+ return sqlite3_reset(pSeek); |
+ } |
+ } |
+ |
+ ctx.pStorage = p; |
+ ctx.iCol = -1; |
+ rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); |
+ for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ |
+ if( pConfig->abUnindexed[iCol-1]==0 ){ |
+ const char *zText; |
+ int nText; |
+ if( pSeek ){ |
+ zText = (const char*)sqlite3_column_text(pSeek, iCol); |
+ nText = sqlite3_column_bytes(pSeek, iCol); |
+ }else{ |
+ zText = (const char*)sqlite3_value_text(apVal[iCol-1]); |
+ nText = sqlite3_value_bytes(apVal[iCol-1]); |
+ } |
+ ctx.szCol = 0; |
+ rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, |
+ zText, nText, (void*)&ctx, fts5StorageInsertCallback |
+ ); |
+ p->aTotalSize[iCol-1] -= (i64)ctx.szCol; |
+ } |
+ } |
+ p->nTotalRow--; |
+ |
+ rc2 = sqlite3_reset(pSeek); |
+ if( rc==SQLITE_OK ) rc = rc2; |
+ return rc; |
+} |
+ |
+ |
+/* |
+** Insert a record into the %_docsize table. Specifically, do: |
+** |
+** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf); |
+** |
+** If there is no %_docsize table (as happens if the columnsize=0 option |
+** is specified when the FTS5 table is created), this function is a no-op. |
+*/ |
+static int fts5StorageInsertDocsize( |
+ Fts5Storage *p, /* Storage module to write to */ |
+ i64 iRowid, /* id value */ |
+ Fts5Buffer *pBuf /* sz value */ |
+){ |
+ int rc = SQLITE_OK; |
+ if( p->pConfig->bColumnsize ){ |
+ sqlite3_stmt *pReplace = 0; |
+ rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pReplace, 1, iRowid); |
+ sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); |
+ sqlite3_step(pReplace); |
+ rc = sqlite3_reset(pReplace); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Load the contents of the "averages" record from disk into the |
+** p->nTotalRow and p->aTotalSize[] variables. If successful, and if |
+** argument bCache is true, set the p->bTotalsValid flag to indicate |
+** that the contents of aTotalSize[] and nTotalRow are valid until |
+** further notice. |
+** |
+** Return SQLITE_OK if successful, or an SQLite error code if an error |
+** occurs. |
+*/ |
+static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){ |
+ int rc = SQLITE_OK; |
+ if( p->bTotalsValid==0 ){ |
+ rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize); |
+ p->bTotalsValid = bCache; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Store the current contents of the p->nTotalRow and p->aTotalSize[] |
+** variables in the "averages" record on disk. |
+** |
+** Return SQLITE_OK if successful, or an SQLite error code if an error |
+** occurs. |
+*/ |
+static int fts5StorageSaveTotals(Fts5Storage *p){ |
+ int nCol = p->pConfig->nCol; |
+ int i; |
+ Fts5Buffer buf; |
+ int rc = SQLITE_OK; |
+ memset(&buf, 0, sizeof(buf)); |
+ |
+ sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow); |
+ for(i=0; i<nCol; i++){ |
+ sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n); |
+ } |
+ sqlite3_free(buf.p); |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Remove a row from the FTS table. |
+*/ |
+static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ |
+ Fts5Config *pConfig = p->pConfig; |
+ int rc; |
+ sqlite3_stmt *pDel = 0; |
+ |
+ assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 ); |
+ rc = fts5StorageLoadTotals(p, 1); |
+ |
+ /* Delete the index records */ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5StorageDeleteFromIndex(p, iDel, apVal); |
+ } |
+ |
+ /* Delete the %_docsize record */ |
+ if( rc==SQLITE_OK && pConfig->bColumnsize ){ |
+ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pDel, 1, iDel); |
+ sqlite3_step(pDel); |
+ rc = sqlite3_reset(pDel); |
+ } |
+ } |
+ |
+ /* Delete the %_content record */ |
+ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_int64(pDel, 1, iDel); |
+ sqlite3_step(pDel); |
+ rc = sqlite3_reset(pDel); |
+ } |
+ } |
+ |
+ /* Write the averages record */ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5StorageSaveTotals(p); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Delete all entries in the FTS5 index. |
+*/ |
+static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ |
+ Fts5Config *pConfig = p->pConfig; |
+ int rc; |
+ |
+ /* Delete the contents of the %_data and %_docsize tables. */ |
+ rc = fts5ExecPrintf(pConfig->db, 0, |
+ "DELETE FROM %Q.'%q_data';" |
+ "DELETE FROM %Q.'%q_idx';", |
+ pConfig->zDb, pConfig->zName, |
+ pConfig->zDb, pConfig->zName |
+ ); |
+ if( rc==SQLITE_OK && pConfig->bColumnsize ){ |
+ rc = fts5ExecPrintf(pConfig->db, 0, |
+ "DELETE FROM %Q.'%q_docsize';", |
+ pConfig->zDb, pConfig->zName |
+ ); |
+ } |
+ |
+ /* Reinitialize the %_data table. This call creates the initial structure |
+ ** and averages records. */ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5IndexReinit(p->pIndex); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); |
+ } |
+ return rc; |
+} |
+ |
+static int sqlite3Fts5StorageRebuild(Fts5Storage *p){ |
+ Fts5Buffer buf = {0,0,0}; |
+ Fts5Config *pConfig = p->pConfig; |
+ sqlite3_stmt *pScan = 0; |
+ Fts5InsertCtx ctx; |
+ int rc; |
+ |
+ memset(&ctx, 0, sizeof(Fts5InsertCtx)); |
+ ctx.pStorage = p; |
+ rc = sqlite3Fts5StorageDeleteAll(p); |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5StorageLoadTotals(p, 1); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); |
+ } |
+ |
+ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ |
+ i64 iRowid = sqlite3_column_int64(pScan, 0); |
+ |
+ sqlite3Fts5BufferZero(&buf); |
+ rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid); |
+ for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){ |
+ ctx.szCol = 0; |
+ if( pConfig->abUnindexed[ctx.iCol]==0 ){ |
+ rc = sqlite3Fts5Tokenize(pConfig, |
+ FTS5_TOKENIZE_DOCUMENT, |
+ (const char*)sqlite3_column_text(pScan, ctx.iCol+1), |
+ sqlite3_column_bytes(pScan, ctx.iCol+1), |
+ (void*)&ctx, |
+ fts5StorageInsertCallback |
+ ); |
+ } |
+ sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); |
+ p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; |
+ } |
+ p->nTotalRow++; |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5StorageInsertDocsize(p, iRowid, &buf); |
+ } |
+ } |
+ sqlite3_free(buf.p); |
+ |
+ /* Write the averages record */ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5StorageSaveTotals(p); |
+ } |
+ return rc; |
+} |
+ |
+static int sqlite3Fts5StorageOptimize(Fts5Storage *p){ |
+ return sqlite3Fts5IndexOptimize(p->pIndex); |
+} |
+ |
+static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){ |
+ return sqlite3Fts5IndexMerge(p->pIndex, nMerge); |
+} |
+ |
+static int sqlite3Fts5StorageReset(Fts5Storage *p){ |
+ return sqlite3Fts5IndexReset(p->pIndex); |
+} |
+ |
+/* |
+** Allocate a new rowid. This is used for "external content" tables when |
+** a NULL value is inserted into the rowid column. The new rowid is allocated |
+** by inserting a dummy row into the %_docsize table. The dummy will be |
+** overwritten later. |
+** |
+** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In |
+** this case the user is required to provide a rowid explicitly. |
+*/ |
+static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){ |
+ int rc = SQLITE_MISMATCH; |
+ if( p->pConfig->bColumnsize ){ |
+ sqlite3_stmt *pReplace = 0; |
+ rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_null(pReplace, 1); |
+ sqlite3_bind_null(pReplace, 2); |
+ sqlite3_step(pReplace); |
+ rc = sqlite3_reset(pReplace); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ *piRowid = sqlite3_last_insert_rowid(p->pConfig->db); |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Insert a new row into the FTS content table. |
+*/ |
+static int sqlite3Fts5StorageContentInsert( |
+ Fts5Storage *p, |
+ sqlite3_value **apVal, |
+ i64 *piRowid |
+){ |
+ Fts5Config *pConfig = p->pConfig; |
+ int rc = SQLITE_OK; |
+ |
+ /* Insert the new row into the %_content table. */ |
+ if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ |
+ if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ |
+ *piRowid = sqlite3_value_int64(apVal[1]); |
+ }else{ |
+ rc = fts5StorageNewRowid(p, piRowid); |
+ } |
+ }else{ |
+ sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */ |
+ int i; /* Counter variable */ |
+ rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); |
+ for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ |
+ rc = sqlite3_bind_value(pInsert, i, apVal[i]); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_step(pInsert); |
+ rc = sqlite3_reset(pInsert); |
+ } |
+ *piRowid = sqlite3_last_insert_rowid(pConfig->db); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Insert new entries into the FTS index and %_docsize table. |
+*/ |
+static int sqlite3Fts5StorageIndexInsert( |
+ Fts5Storage *p, |
+ sqlite3_value **apVal, |
+ i64 iRowid |
+){ |
+ Fts5Config *pConfig = p->pConfig; |
+ int rc = SQLITE_OK; /* Return code */ |
+ Fts5InsertCtx ctx; /* Tokenization callback context object */ |
+ Fts5Buffer buf; /* Buffer used to build up %_docsize blob */ |
+ |
+ memset(&buf, 0, sizeof(Fts5Buffer)); |
+ ctx.pStorage = p; |
+ rc = fts5StorageLoadTotals(p, 1); |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid); |
+ } |
+ for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){ |
+ ctx.szCol = 0; |
+ if( pConfig->abUnindexed[ctx.iCol]==0 ){ |
+ rc = sqlite3Fts5Tokenize(pConfig, |
+ FTS5_TOKENIZE_DOCUMENT, |
+ (const char*)sqlite3_value_text(apVal[ctx.iCol+2]), |
+ sqlite3_value_bytes(apVal[ctx.iCol+2]), |
+ (void*)&ctx, |
+ fts5StorageInsertCallback |
+ ); |
+ } |
+ sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); |
+ p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; |
+ } |
+ p->nTotalRow++; |
+ |
+ /* Write the %_docsize record */ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5StorageInsertDocsize(p, iRowid, &buf); |
+ } |
+ sqlite3_free(buf.p); |
+ |
+ /* Write the averages record */ |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5StorageSaveTotals(p); |
+ } |
+ |
+ return rc; |
+} |
+ |
+static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){ |
+ Fts5Config *pConfig = p->pConfig; |
+ char *zSql; |
+ int rc; |
+ |
+ zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'", |
+ pConfig->zDb, pConfig->zName, zSuffix |
+ ); |
+ if( zSql==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ sqlite3_stmt *pCnt = 0; |
+ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0); |
+ if( rc==SQLITE_OK ){ |
+ if( SQLITE_ROW==sqlite3_step(pCnt) ){ |
+ *pnRow = sqlite3_column_int64(pCnt, 0); |
+ } |
+ rc = sqlite3_finalize(pCnt); |
+ } |
+ } |
+ |
+ sqlite3_free(zSql); |
+ return rc; |
+} |
+ |
+/* |
+** Context object used by sqlite3Fts5StorageIntegrity(). |
+*/ |
+typedef struct Fts5IntegrityCtx Fts5IntegrityCtx; |
+struct Fts5IntegrityCtx { |
+ i64 iRowid; |
+ int iCol; |
+ int szCol; |
+ u64 cksum; |
+ Fts5Termset *pTermset; |
+ Fts5Config *pConfig; |
+}; |
+ |
+ |
+/* |
+** Tokenization callback used by integrity check. |
+*/ |
+static int fts5StorageIntegrityCallback( |
+ void *pContext, /* Pointer to Fts5IntegrityCtx object */ |
+ int tflags, |
+ const char *pToken, /* Buffer containing token */ |
+ int nToken, /* Size of token in bytes */ |
+ int iUnused1, /* Start offset of token */ |
+ int iUnused2 /* End offset of token */ |
+){ |
+ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; |
+ Fts5Termset *pTermset = pCtx->pTermset; |
+ int bPresent; |
+ int ii; |
+ int rc = SQLITE_OK; |
+ int iPos; |
+ int iCol; |
+ |
+ UNUSED_PARAM2(iUnused1, iUnused2); |
+ if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; |
+ |
+ if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ |
+ pCtx->szCol++; |
+ } |
+ |
+ switch( pCtx->pConfig->eDetail ){ |
+ case FTS5_DETAIL_FULL: |
+ iPos = pCtx->szCol-1; |
+ iCol = pCtx->iCol; |
+ break; |
+ |
+ case FTS5_DETAIL_COLUMNS: |
+ iPos = pCtx->iCol; |
+ iCol = 0; |
+ break; |
+ |
+ default: |
+ assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE ); |
+ iPos = 0; |
+ iCol = 0; |
+ break; |
+ } |
+ |
+ rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent); |
+ if( rc==SQLITE_OK && bPresent==0 ){ |
+ pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( |
+ pCtx->iRowid, iCol, iPos, 0, pToken, nToken |
+ ); |
+ } |
+ |
+ for(ii=0; rc==SQLITE_OK && ii<pCtx->pConfig->nPrefix; ii++){ |
+ const int nChar = pCtx->pConfig->aPrefix[ii]; |
+ int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); |
+ if( nByte ){ |
+ rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent); |
+ if( bPresent==0 ){ |
+ pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( |
+ pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte |
+ ); |
+ } |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Check that the contents of the FTS index match that of the %_content |
+** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return |
+** some other SQLite error code if an error occurs while attempting to |
+** determine this. |
+*/ |
+static int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ |
+ Fts5Config *pConfig = p->pConfig; |
+ int rc; /* Return code */ |
+ int *aColSize; /* Array of size pConfig->nCol */ |
+ i64 *aTotalSize; /* Array of size pConfig->nCol */ |
+ Fts5IntegrityCtx ctx; |
+ sqlite3_stmt *pScan; |
+ |
+ memset(&ctx, 0, sizeof(Fts5IntegrityCtx)); |
+ ctx.pConfig = p->pConfig; |
+ aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64))); |
+ if( !aTotalSize ) return SQLITE_NOMEM; |
+ aColSize = (int*)&aTotalSize[pConfig->nCol]; |
+ memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol); |
+ |
+ /* Generate the expected index checksum based on the contents of the |
+ ** %_content table. This block stores the checksum in ctx.cksum. */ |
+ rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); |
+ if( rc==SQLITE_OK ){ |
+ int rc2; |
+ while( SQLITE_ROW==sqlite3_step(pScan) ){ |
+ int i; |
+ ctx.iRowid = sqlite3_column_int64(pScan, 0); |
+ ctx.szCol = 0; |
+ if( pConfig->bColumnsize ){ |
+ rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); |
+ } |
+ if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){ |
+ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); |
+ } |
+ for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ |
+ if( pConfig->abUnindexed[i] ) continue; |
+ ctx.iCol = i; |
+ ctx.szCol = 0; |
+ if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ |
+ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5Tokenize(pConfig, |
+ FTS5_TOKENIZE_DOCUMENT, |
+ (const char*)sqlite3_column_text(pScan, i+1), |
+ sqlite3_column_bytes(pScan, i+1), |
+ (void*)&ctx, |
+ fts5StorageIntegrityCallback |
+ ); |
+ } |
+ if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ |
+ rc = FTS5_CORRUPT; |
+ } |
+ aTotalSize[i] += ctx.szCol; |
+ if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ |
+ sqlite3Fts5TermsetFree(ctx.pTermset); |
+ ctx.pTermset = 0; |
+ } |
+ } |
+ sqlite3Fts5TermsetFree(ctx.pTermset); |
+ ctx.pTermset = 0; |
+ |
+ if( rc!=SQLITE_OK ) break; |
+ } |
+ rc2 = sqlite3_reset(pScan); |
+ if( rc==SQLITE_OK ) rc = rc2; |
+ } |
+ |
+ /* Test that the "totals" (sometimes called "averages") record looks Ok */ |
+ if( rc==SQLITE_OK ){ |
+ int i; |
+ rc = fts5StorageLoadTotals(p, 0); |
+ for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ |
+ if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT; |
+ } |
+ } |
+ |
+ /* Check that the %_docsize and %_content tables contain the expected |
+ ** number of rows. */ |
+ if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ |
+ i64 nRow = 0; |
+ rc = fts5StorageCount(p, "content", &nRow); |
+ if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; |
+ } |
+ if( rc==SQLITE_OK && pConfig->bColumnsize ){ |
+ i64 nRow = 0; |
+ rc = fts5StorageCount(p, "docsize", &nRow); |
+ if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; |
+ } |
+ |
+ /* Pass the expected checksum down to the FTS index module. It will |
+ ** verify, amongst other things, that it matches the checksum generated by |
+ ** inspecting the index itself. */ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum); |
+ } |
+ |
+ sqlite3_free(aTotalSize); |
+ return rc; |
+} |
+ |
+/* |
+** Obtain an SQLite statement handle that may be used to read data from the |
+** %_content table. |
+*/ |
+static int sqlite3Fts5StorageStmt( |
+ Fts5Storage *p, |
+ int eStmt, |
+ sqlite3_stmt **pp, |
+ char **pzErrMsg |
+){ |
+ int rc; |
+ assert( eStmt==FTS5_STMT_SCAN_ASC |
+ || eStmt==FTS5_STMT_SCAN_DESC |
+ || eStmt==FTS5_STMT_LOOKUP |
+ ); |
+ rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg); |
+ if( rc==SQLITE_OK ){ |
+ assert( p->aStmt[eStmt]==*pp ); |
+ p->aStmt[eStmt] = 0; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Release an SQLite statement handle obtained via an earlier call to |
+** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function |
+** must match that passed to the sqlite3Fts5StorageStmt() call. |
+*/ |
+static void sqlite3Fts5StorageStmtRelease( |
+ Fts5Storage *p, |
+ int eStmt, |
+ sqlite3_stmt *pStmt |
+){ |
+ assert( eStmt==FTS5_STMT_SCAN_ASC |
+ || eStmt==FTS5_STMT_SCAN_DESC |
+ || eStmt==FTS5_STMT_LOOKUP |
+ ); |
+ if( p->aStmt[eStmt]==0 ){ |
+ sqlite3_reset(pStmt); |
+ p->aStmt[eStmt] = pStmt; |
+ }else{ |
+ sqlite3_finalize(pStmt); |
+ } |
+} |
+ |
+static int fts5StorageDecodeSizeArray( |
+ int *aCol, int nCol, /* Array to populate */ |
+ const u8 *aBlob, int nBlob /* Record to read varints from */ |
+){ |
+ int i; |
+ int iOff = 0; |
+ for(i=0; i<nCol; i++){ |
+ if( iOff>=nBlob ) return 1; |
+ iOff += fts5GetVarint32(&aBlob[iOff], aCol[i]); |
+ } |
+ return (iOff!=nBlob); |
+} |
+ |
+/* |
+** Argument aCol points to an array of integers containing one entry for |
+** each table column. This function reads the %_docsize record for the |
+** specified rowid and populates aCol[] with the results. |
+** |
+** An SQLite error code is returned if an error occurs, or SQLITE_OK |
+** otherwise. |
+*/ |
+static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){ |
+ int nCol = p->pConfig->nCol; /* Number of user columns in table */ |
+ sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */ |
+ int rc; /* Return Code */ |
+ |
+ assert( p->pConfig->bColumnsize ); |
+ rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); |
+ if( rc==SQLITE_OK ){ |
+ int bCorrupt = 1; |
+ sqlite3_bind_int64(pLookup, 1, iRowid); |
+ if( SQLITE_ROW==sqlite3_step(pLookup) ){ |
+ const u8 *aBlob = sqlite3_column_blob(pLookup, 0); |
+ int nBlob = sqlite3_column_bytes(pLookup, 0); |
+ if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){ |
+ bCorrupt = 0; |
+ } |
+ } |
+ rc = sqlite3_reset(pLookup); |
+ if( bCorrupt && rc==SQLITE_OK ){ |
+ rc = FTS5_CORRUPT; |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){ |
+ int rc = fts5StorageLoadTotals(p, 0); |
+ if( rc==SQLITE_OK ){ |
+ *pnToken = 0; |
+ if( iCol<0 ){ |
+ int i; |
+ for(i=0; i<p->pConfig->nCol; i++){ |
+ *pnToken += p->aTotalSize[i]; |
+ } |
+ }else if( iCol<p->pConfig->nCol ){ |
+ *pnToken = p->aTotalSize[iCol]; |
+ }else{ |
+ rc = SQLITE_RANGE; |
+ } |
+ } |
+ return rc; |
+} |
+ |
+static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){ |
+ int rc = fts5StorageLoadTotals(p, 0); |
+ if( rc==SQLITE_OK ){ |
+ *pnRow = p->nTotalRow; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Flush any data currently held in-memory to disk. |
+*/ |
+static int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){ |
+ if( bCommit && p->bTotalsValid ){ |
+ int rc = fts5StorageSaveTotals(p); |
+ p->bTotalsValid = 0; |
+ if( rc!=SQLITE_OK ) return rc; |
+ } |
+ return sqlite3Fts5IndexSync(p->pIndex, bCommit); |
+} |
+ |
+static int sqlite3Fts5StorageRollback(Fts5Storage *p){ |
+ p->bTotalsValid = 0; |
+ return sqlite3Fts5IndexRollback(p->pIndex); |
+} |
+ |
+static int sqlite3Fts5StorageConfigValue( |
+ Fts5Storage *p, |
+ const char *z, |
+ sqlite3_value *pVal, |
+ int iVal |
+){ |
+ sqlite3_stmt *pReplace = 0; |
+ int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC); |
+ if( pVal ){ |
+ sqlite3_bind_value(pReplace, 2, pVal); |
+ }else{ |
+ sqlite3_bind_int(pReplace, 2, iVal); |
+ } |
+ sqlite3_step(pReplace); |
+ rc = sqlite3_reset(pReplace); |
+ } |
+ if( rc==SQLITE_OK && pVal ){ |
+ int iNew = p->pConfig->iCookie + 1; |
+ rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew); |
+ if( rc==SQLITE_OK ){ |
+ p->pConfig->iCookie = iNew; |
+ } |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** 2014 May 31 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+*/ |
+ |
+ |
+/* #include "fts5Int.h" */ |
+ |
+/************************************************************************** |
+** Start of ascii tokenizer implementation. |
+*/ |
+ |
+/* |
+** For tokenizers with no "unicode" modifier, the set of token characters |
+** is the same as the set of ASCII range alphanumeric characters. |
+*/ |
+static unsigned char aAsciiTokenChar[128] = { |
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */ |
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */ |
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */ |
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */ |
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */ |
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */ |
+}; |
+ |
+typedef struct AsciiTokenizer AsciiTokenizer; |
+struct AsciiTokenizer { |
+ unsigned char aTokenChar[128]; |
+}; |
+ |
+static void fts5AsciiAddExceptions( |
+ AsciiTokenizer *p, |
+ const char *zArg, |
+ int bTokenChars |
+){ |
+ int i; |
+ for(i=0; zArg[i]; i++){ |
+ if( (zArg[i] & 0x80)==0 ){ |
+ p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars; |
+ } |
+ } |
+} |
+ |
+/* |
+** Delete a "ascii" tokenizer. |
+*/ |
+static void fts5AsciiDelete(Fts5Tokenizer *p){ |
+ sqlite3_free(p); |
+} |
+ |
+/* |
+** Create an "ascii" tokenizer. |
+*/ |
+static int fts5AsciiCreate( |
+ void *pUnused, |
+ const char **azArg, int nArg, |
+ Fts5Tokenizer **ppOut |
+){ |
+ int rc = SQLITE_OK; |
+ AsciiTokenizer *p = 0; |
+ UNUSED_PARAM(pUnused); |
+ if( nArg%2 ){ |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ p = sqlite3_malloc(sizeof(AsciiTokenizer)); |
+ if( p==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ int i; |
+ memset(p, 0, sizeof(AsciiTokenizer)); |
+ memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); |
+ for(i=0; rc==SQLITE_OK && i<nArg; i+=2){ |
+ const char *zArg = azArg[i+1]; |
+ if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ |
+ fts5AsciiAddExceptions(p, zArg, 1); |
+ }else |
+ if( 0==sqlite3_stricmp(azArg[i], "separators") ){ |
+ fts5AsciiAddExceptions(p, zArg, 0); |
+ }else{ |
+ rc = SQLITE_ERROR; |
+ } |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ fts5AsciiDelete((Fts5Tokenizer*)p); |
+ p = 0; |
+ } |
+ } |
+ } |
+ |
+ *ppOut = (Fts5Tokenizer*)p; |
+ return rc; |
+} |
+ |
+ |
+static void asciiFold(char *aOut, const char *aIn, int nByte){ |
+ int i; |
+ for(i=0; i<nByte; i++){ |
+ char c = aIn[i]; |
+ if( c>='A' && c<='Z' ) c += 32; |
+ aOut[i] = c; |
+ } |
+} |
+ |
+/* |
+** Tokenize some text using the ascii tokenizer. |
+*/ |
+static int fts5AsciiTokenize( |
+ Fts5Tokenizer *pTokenizer, |
+ void *pCtx, |
+ int iUnused, |
+ const char *pText, int nText, |
+ int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) |
+){ |
+ AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer; |
+ int rc = SQLITE_OK; |
+ int ie; |
+ int is = 0; |
+ |
+ char aFold[64]; |
+ int nFold = sizeof(aFold); |
+ char *pFold = aFold; |
+ unsigned char *a = p->aTokenChar; |
+ |
+ UNUSED_PARAM(iUnused); |
+ |
+ while( is<nText && rc==SQLITE_OK ){ |
+ int nByte; |
+ |
+ /* Skip any leading divider characters. */ |
+ while( is<nText && ((pText[is]&0x80)==0 && a[(int)pText[is]]==0) ){ |
+ is++; |
+ } |
+ if( is==nText ) break; |
+ |
+ /* Count the token characters */ |
+ ie = is+1; |
+ while( ie<nText && ((pText[ie]&0x80) || a[(int)pText[ie]] ) ){ |
+ ie++; |
+ } |
+ |
+ /* Fold to lower case */ |
+ nByte = ie-is; |
+ if( nByte>nFold ){ |
+ if( pFold!=aFold ) sqlite3_free(pFold); |
+ pFold = sqlite3_malloc(nByte*2); |
+ if( pFold==0 ){ |
+ rc = SQLITE_NOMEM; |
+ break; |
+ } |
+ nFold = nByte*2; |
+ } |
+ asciiFold(pFold, &pText[is], nByte); |
+ |
+ /* Invoke the token callback */ |
+ rc = xToken(pCtx, 0, pFold, nByte, is, ie); |
+ is = ie+1; |
+ } |
+ |
+ if( pFold!=aFold ) sqlite3_free(pFold); |
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK; |
+ return rc; |
+} |
+ |
+/************************************************************************** |
+** Start of unicode61 tokenizer implementation. |
+*/ |
+ |
+ |
+/* |
+** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied |
+** from the sqlite3 source file utf.c. If this file is compiled as part |
+** of the amalgamation, they are not required. |
+*/ |
+#ifndef SQLITE_AMALGAMATION |
+ |
+static const unsigned char sqlite3Utf8Trans1[] = { |
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, |
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, |
+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, |
+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, |
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, |
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, |
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, |
+ 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, |
+}; |
+ |
+#define READ_UTF8(zIn, zTerm, c) \ |
+ c = *(zIn++); \ |
+ if( c>=0xc0 ){ \ |
+ c = sqlite3Utf8Trans1[c-0xc0]; \ |
+ while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ |
+ c = (c<<6) + (0x3f & *(zIn++)); \ |
+ } \ |
+ if( c<0x80 \ |
+ || (c&0xFFFFF800)==0xD800 \ |
+ || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ |
+ } |
+ |
+ |
+#define WRITE_UTF8(zOut, c) { \ |
+ if( c<0x00080 ){ \ |
+ *zOut++ = (unsigned char)(c&0xFF); \ |
+ } \ |
+ else if( c<0x00800 ){ \ |
+ *zOut++ = 0xC0 + (unsigned char)((c>>6)&0x1F); \ |
+ *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ |
+ } \ |
+ else if( c<0x10000 ){ \ |
+ *zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \ |
+ *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ |
+ *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ |
+ }else{ \ |
+ *zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \ |
+ *zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \ |
+ *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ |
+ *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ |
+ } \ |
+} |
+ |
+#endif /* ifndef SQLITE_AMALGAMATION */ |
+ |
+typedef struct Unicode61Tokenizer Unicode61Tokenizer; |
+struct Unicode61Tokenizer { |
+ unsigned char aTokenChar[128]; /* ASCII range token characters */ |
+ char *aFold; /* Buffer to fold text into */ |
+ int nFold; /* Size of aFold[] in bytes */ |
+ int bRemoveDiacritic; /* True if remove_diacritics=1 is set */ |
+ int nException; |
+ int *aiException; |
+}; |
+ |
+static int fts5UnicodeAddExceptions( |
+ Unicode61Tokenizer *p, /* Tokenizer object */ |
+ const char *z, /* Characters to treat as exceptions */ |
+ int bTokenChars /* 1 for 'tokenchars', 0 for 'separators' */ |
+){ |
+ int rc = SQLITE_OK; |
+ int n = (int)strlen(z); |
+ int *aNew; |
+ |
+ if( n>0 ){ |
+ aNew = (int*)sqlite3_realloc(p->aiException, (n+p->nException)*sizeof(int)); |
+ if( aNew ){ |
+ int nNew = p->nException; |
+ const unsigned char *zCsr = (const unsigned char*)z; |
+ const unsigned char *zTerm = (const unsigned char*)&z[n]; |
+ while( zCsr<zTerm ){ |
+ int iCode; |
+ int bToken; |
+ READ_UTF8(zCsr, zTerm, iCode); |
+ if( iCode<128 ){ |
+ p->aTokenChar[iCode] = (unsigned char)bTokenChars; |
+ }else{ |
+ bToken = sqlite3Fts5UnicodeIsalnum(iCode); |
+ assert( (bToken==0 || bToken==1) ); |
+ assert( (bTokenChars==0 || bTokenChars==1) ); |
+ if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){ |
+ int i; |
+ for(i=0; i<nNew; i++){ |
+ if( aNew[i]>iCode ) break; |
+ } |
+ memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int)); |
+ aNew[i] = iCode; |
+ nNew++; |
+ } |
+ } |
+ } |
+ p->aiException = aNew; |
+ p->nException = nNew; |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ } |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** Return true if the p->aiException[] array contains the value iCode. |
+*/ |
+static int fts5UnicodeIsException(Unicode61Tokenizer *p, int iCode){ |
+ if( p->nException>0 ){ |
+ int *a = p->aiException; |
+ int iLo = 0; |
+ int iHi = p->nException-1; |
+ |
+ while( iHi>=iLo ){ |
+ int iTest = (iHi + iLo) / 2; |
+ if( iCode==a[iTest] ){ |
+ return 1; |
+ }else if( iCode>a[iTest] ){ |
+ iLo = iTest+1; |
+ }else{ |
+ iHi = iTest-1; |
+ } |
+ } |
+ } |
+ |
+ return 0; |
+} |
+ |
+/* |
+** Delete a "unicode61" tokenizer. |
+*/ |
+static void fts5UnicodeDelete(Fts5Tokenizer *pTok){ |
+ if( pTok ){ |
+ Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok; |
+ sqlite3_free(p->aiException); |
+ sqlite3_free(p->aFold); |
+ sqlite3_free(p); |
+ } |
+ return; |
+} |
+ |
+/* |
+** Create a "unicode61" tokenizer. |
+*/ |
+static int fts5UnicodeCreate( |
+ void *pUnused, |
+ const char **azArg, int nArg, |
+ Fts5Tokenizer **ppOut |
+){ |
+ int rc = SQLITE_OK; /* Return code */ |
+ Unicode61Tokenizer *p = 0; /* New tokenizer object */ |
+ |
+ UNUSED_PARAM(pUnused); |
+ |
+ if( nArg%2 ){ |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); |
+ if( p ){ |
+ int i; |
+ memset(p, 0, sizeof(Unicode61Tokenizer)); |
+ memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); |
+ p->bRemoveDiacritic = 1; |
+ p->nFold = 64; |
+ p->aFold = sqlite3_malloc(p->nFold * sizeof(char)); |
+ if( p->aFold==0 ){ |
+ rc = SQLITE_NOMEM; |
+ } |
+ for(i=0; rc==SQLITE_OK && i<nArg; i+=2){ |
+ const char *zArg = azArg[i+1]; |
+ if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ |
+ if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){ |
+ rc = SQLITE_ERROR; |
+ } |
+ p->bRemoveDiacritic = (zArg[0]=='1'); |
+ }else |
+ if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ |
+ rc = fts5UnicodeAddExceptions(p, zArg, 1); |
+ }else |
+ if( 0==sqlite3_stricmp(azArg[i], "separators") ){ |
+ rc = fts5UnicodeAddExceptions(p, zArg, 0); |
+ }else{ |
+ rc = SQLITE_ERROR; |
+ } |
+ } |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ } |
+ if( rc!=SQLITE_OK ){ |
+ fts5UnicodeDelete((Fts5Tokenizer*)p); |
+ p = 0; |
+ } |
+ *ppOut = (Fts5Tokenizer*)p; |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** Return true if, for the purposes of tokenizing with the tokenizer |
+** passed as the first argument, codepoint iCode is considered a token |
+** character (not a separator). |
+*/ |
+static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){ |
+ assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); |
+ return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode); |
+} |
+ |
+static int fts5UnicodeTokenize( |
+ Fts5Tokenizer *pTokenizer, |
+ void *pCtx, |
+ int iUnused, |
+ const char *pText, int nText, |
+ int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) |
+){ |
+ Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer; |
+ int rc = SQLITE_OK; |
+ unsigned char *a = p->aTokenChar; |
+ |
+ unsigned char *zTerm = (unsigned char*)&pText[nText]; |
+ unsigned char *zCsr = (unsigned char *)pText; |
+ |
+ /* Output buffer */ |
+ char *aFold = p->aFold; |
+ int nFold = p->nFold; |
+ const char *pEnd = &aFold[nFold-6]; |
+ |
+ UNUSED_PARAM(iUnused); |
+ |
+ /* Each iteration of this loop gobbles up a contiguous run of separators, |
+ ** then the next token. */ |
+ while( rc==SQLITE_OK ){ |
+ int iCode; /* non-ASCII codepoint read from input */ |
+ char *zOut = aFold; |
+ int is; |
+ int ie; |
+ |
+ /* Skip any separator characters. */ |
+ while( 1 ){ |
+ if( zCsr>=zTerm ) goto tokenize_done; |
+ if( *zCsr & 0x80 ) { |
+ /* A character outside of the ascii range. Skip past it if it is |
+ ** a separator character. Or break out of the loop if it is not. */ |
+ is = zCsr - (unsigned char*)pText; |
+ READ_UTF8(zCsr, zTerm, iCode); |
+ if( fts5UnicodeIsAlnum(p, iCode) ){ |
+ goto non_ascii_tokenchar; |
+ } |
+ }else{ |
+ if( a[*zCsr] ){ |
+ is = zCsr - (unsigned char*)pText; |
+ goto ascii_tokenchar; |
+ } |
+ zCsr++; |
+ } |
+ } |
+ |
+ /* Run through the tokenchars. Fold them into the output buffer along |
+ ** the way. */ |
+ while( zCsr<zTerm ){ |
+ |
+ /* Grow the output buffer so that there is sufficient space to fit the |
+ ** largest possible utf-8 character. */ |
+ if( zOut>pEnd ){ |
+ aFold = sqlite3_malloc(nFold*2); |
+ if( aFold==0 ){ |
+ rc = SQLITE_NOMEM; |
+ goto tokenize_done; |
+ } |
+ zOut = &aFold[zOut - p->aFold]; |
+ memcpy(aFold, p->aFold, nFold); |
+ sqlite3_free(p->aFold); |
+ p->aFold = aFold; |
+ p->nFold = nFold = nFold*2; |
+ pEnd = &aFold[nFold-6]; |
+ } |
+ |
+ if( *zCsr & 0x80 ){ |
+ /* An non-ascii-range character. Fold it into the output buffer if |
+ ** it is a token character, or break out of the loop if it is not. */ |
+ READ_UTF8(zCsr, zTerm, iCode); |
+ if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){ |
+ non_ascii_tokenchar: |
+ iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic); |
+ if( iCode ) WRITE_UTF8(zOut, iCode); |
+ }else{ |
+ break; |
+ } |
+ }else if( a[*zCsr]==0 ){ |
+ /* An ascii-range separator character. End of token. */ |
+ break; |
+ }else{ |
+ ascii_tokenchar: |
+ if( *zCsr>='A' && *zCsr<='Z' ){ |
+ *zOut++ = *zCsr + 32; |
+ }else{ |
+ *zOut++ = *zCsr; |
+ } |
+ zCsr++; |
+ } |
+ ie = zCsr - (unsigned char*)pText; |
+ } |
+ |
+ /* Invoke the token callback */ |
+ rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie); |
+ } |
+ |
+ tokenize_done: |
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK; |
+ return rc; |
+} |
+ |
+/************************************************************************** |
+** Start of porter stemmer implementation. |
+*/ |
+ |
+/* Any tokens larger than this (in bytes) are passed through without |
+** stemming. */ |
+#define FTS5_PORTER_MAX_TOKEN 64 |
+ |
+typedef struct PorterTokenizer PorterTokenizer; |
+struct PorterTokenizer { |
+ fts5_tokenizer tokenizer; /* Parent tokenizer module */ |
+ Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */ |
+ char aBuf[FTS5_PORTER_MAX_TOKEN + 64]; |
+}; |
+ |
+/* |
+** Delete a "porter" tokenizer. |
+*/ |
+static void fts5PorterDelete(Fts5Tokenizer *pTok){ |
+ if( pTok ){ |
+ PorterTokenizer *p = (PorterTokenizer*)pTok; |
+ if( p->pTokenizer ){ |
+ p->tokenizer.xDelete(p->pTokenizer); |
+ } |
+ sqlite3_free(p); |
+ } |
+} |
+ |
+/* |
+** Create a "porter" tokenizer. |
+*/ |
+static int fts5PorterCreate( |
+ void *pCtx, |
+ const char **azArg, int nArg, |
+ Fts5Tokenizer **ppOut |
+){ |
+ fts5_api *pApi = (fts5_api*)pCtx; |
+ int rc = SQLITE_OK; |
+ PorterTokenizer *pRet; |
+ void *pUserdata = 0; |
+ const char *zBase = "unicode61"; |
+ |
+ if( nArg>0 ){ |
+ zBase = azArg[0]; |
+ } |
+ |
+ pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); |
+ if( pRet ){ |
+ memset(pRet, 0, sizeof(PorterTokenizer)); |
+ rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer); |
+ }else{ |
+ rc = SQLITE_NOMEM; |
+ } |
+ if( rc==SQLITE_OK ){ |
+ int nArg2 = (nArg>0 ? nArg-1 : 0); |
+ const char **azArg2 = (nArg2 ? &azArg[1] : 0); |
+ rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer); |
+ } |
+ |
+ if( rc!=SQLITE_OK ){ |
+ fts5PorterDelete((Fts5Tokenizer*)pRet); |
+ pRet = 0; |
+ } |
+ *ppOut = (Fts5Tokenizer*)pRet; |
+ return rc; |
+} |
+ |
+typedef struct PorterContext PorterContext; |
+struct PorterContext { |
+ void *pCtx; |
+ int (*xToken)(void*, int, const char*, int, int, int); |
+ char *aBuf; |
+}; |
+ |
+typedef struct PorterRule PorterRule; |
+struct PorterRule { |
+ const char *zSuffix; |
+ int nSuffix; |
+ int (*xCond)(char *zStem, int nStem); |
+ const char *zOutput; |
+ int nOutput; |
+}; |
+ |
+#if 0 |
+static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){ |
+ int ret = -1; |
+ int nBuf = *pnBuf; |
+ PorterRule *p; |
+ |
+ for(p=aRule; p->zSuffix; p++){ |
+ assert( strlen(p->zSuffix)==p->nSuffix ); |
+ assert( strlen(p->zOutput)==p->nOutput ); |
+ if( nBuf<p->nSuffix ) continue; |
+ if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break; |
+ } |
+ |
+ if( p->zSuffix ){ |
+ int nStem = nBuf - p->nSuffix; |
+ if( p->xCond==0 || p->xCond(aBuf, nStem) ){ |
+ memcpy(&aBuf[nStem], p->zOutput, p->nOutput); |
+ *pnBuf = nStem + p->nOutput; |
+ ret = p - aRule; |
+ } |
+ } |
+ |
+ return ret; |
+} |
+#endif |
+ |
+static int fts5PorterIsVowel(char c, int bYIsVowel){ |
+ return ( |
+ c=='a' || c=='e' || c=='i' || c=='o' || c=='u' || (bYIsVowel && c=='y') |
+ ); |
+} |
+ |
+static int fts5PorterGobbleVC(char *zStem, int nStem, int bPrevCons){ |
+ int i; |
+ int bCons = bPrevCons; |
+ |
+ /* Scan for a vowel */ |
+ for(i=0; i<nStem; i++){ |
+ if( 0==(bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) break; |
+ } |
+ |
+ /* Scan for a consonent */ |
+ for(i++; i<nStem; i++){ |
+ if( (bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) return i+1; |
+ } |
+ return 0; |
+} |
+ |
+/* porter rule condition: (m > 0) */ |
+static int fts5Porter_MGt0(char *zStem, int nStem){ |
+ return !!fts5PorterGobbleVC(zStem, nStem, 0); |
+} |
+ |
+/* porter rule condition: (m > 1) */ |
+static int fts5Porter_MGt1(char *zStem, int nStem){ |
+ int n; |
+ n = fts5PorterGobbleVC(zStem, nStem, 0); |
+ if( n && fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){ |
+ return 1; |
+ } |
+ return 0; |
+} |
+ |
+/* porter rule condition: (m = 1) */ |
+static int fts5Porter_MEq1(char *zStem, int nStem){ |
+ int n; |
+ n = fts5PorterGobbleVC(zStem, nStem, 0); |
+ if( n && 0==fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){ |
+ return 1; |
+ } |
+ return 0; |
+} |
+ |
+/* porter rule condition: (*o) */ |
+static int fts5Porter_Ostar(char *zStem, int nStem){ |
+ if( zStem[nStem-1]=='w' || zStem[nStem-1]=='x' || zStem[nStem-1]=='y' ){ |
+ return 0; |
+ }else{ |
+ int i; |
+ int mask = 0; |
+ int bCons = 0; |
+ for(i=0; i<nStem; i++){ |
+ bCons = !fts5PorterIsVowel(zStem[i], bCons); |
+ assert( bCons==0 || bCons==1 ); |
+ mask = (mask << 1) + bCons; |
+ } |
+ return ((mask & 0x0007)==0x0005); |
+ } |
+} |
+ |
+/* porter rule condition: (m > 1 and (*S or *T)) */ |
+static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){ |
+ assert( nStem>0 ); |
+ return (zStem[nStem-1]=='s' || zStem[nStem-1]=='t') |
+ && fts5Porter_MGt1(zStem, nStem); |
+} |
+ |
+/* porter rule condition: (*v*) */ |
+static int fts5Porter_Vowel(char *zStem, int nStem){ |
+ int i; |
+ for(i=0; i<nStem; i++){ |
+ if( fts5PorterIsVowel(zStem[i], i>0) ){ |
+ return 1; |
+ } |
+ } |
+ return 0; |
+} |
+ |
+ |
+/************************************************************************** |
+*************************************************************************** |
+** GENERATED CODE STARTS HERE (mkportersteps.tcl) |
+*/ |
+ |
+static int fts5PorterStep4(char *aBuf, int *pnBuf){ |
+ int ret = 0; |
+ int nBuf = *pnBuf; |
+ switch( aBuf[nBuf-2] ){ |
+ |
+ case 'a': |
+ if( nBuf>2 && 0==memcmp("al", &aBuf[nBuf-2], 2) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-2) ){ |
+ *pnBuf = nBuf - 2; |
+ } |
+ } |
+ break; |
+ |
+ case 'c': |
+ if( nBuf>4 && 0==memcmp("ance", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-4) ){ |
+ *pnBuf = nBuf - 4; |
+ } |
+ }else if( nBuf>4 && 0==memcmp("ence", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-4) ){ |
+ *pnBuf = nBuf - 4; |
+ } |
+ } |
+ break; |
+ |
+ case 'e': |
+ if( nBuf>2 && 0==memcmp("er", &aBuf[nBuf-2], 2) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-2) ){ |
+ *pnBuf = nBuf - 2; |
+ } |
+ } |
+ break; |
+ |
+ case 'i': |
+ if( nBuf>2 && 0==memcmp("ic", &aBuf[nBuf-2], 2) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-2) ){ |
+ *pnBuf = nBuf - 2; |
+ } |
+ } |
+ break; |
+ |
+ case 'l': |
+ if( nBuf>4 && 0==memcmp("able", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-4) ){ |
+ *pnBuf = nBuf - 4; |
+ } |
+ }else if( nBuf>4 && 0==memcmp("ible", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-4) ){ |
+ *pnBuf = nBuf - 4; |
+ } |
+ } |
+ break; |
+ |
+ case 'n': |
+ if( nBuf>3 && 0==memcmp("ant", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ |
+ *pnBuf = nBuf - 3; |
+ } |
+ }else if( nBuf>5 && 0==memcmp("ement", &aBuf[nBuf-5], 5) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-5) ){ |
+ *pnBuf = nBuf - 5; |
+ } |
+ }else if( nBuf>4 && 0==memcmp("ment", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-4) ){ |
+ *pnBuf = nBuf - 4; |
+ } |
+ }else if( nBuf>3 && 0==memcmp("ent", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ |
+ *pnBuf = nBuf - 3; |
+ } |
+ } |
+ break; |
+ |
+ case 'o': |
+ if( nBuf>3 && 0==memcmp("ion", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt1_and_S_or_T(aBuf, nBuf-3) ){ |
+ *pnBuf = nBuf - 3; |
+ } |
+ }else if( nBuf>2 && 0==memcmp("ou", &aBuf[nBuf-2], 2) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-2) ){ |
+ *pnBuf = nBuf - 2; |
+ } |
+ } |
+ break; |
+ |
+ case 's': |
+ if( nBuf>3 && 0==memcmp("ism", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ |
+ *pnBuf = nBuf - 3; |
+ } |
+ } |
+ break; |
+ |
+ case 't': |
+ if( nBuf>3 && 0==memcmp("ate", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ |
+ *pnBuf = nBuf - 3; |
+ } |
+ }else if( nBuf>3 && 0==memcmp("iti", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ |
+ *pnBuf = nBuf - 3; |
+ } |
+ } |
+ break; |
+ |
+ case 'u': |
+ if( nBuf>3 && 0==memcmp("ous", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ |
+ *pnBuf = nBuf - 3; |
+ } |
+ } |
+ break; |
+ |
+ case 'v': |
+ if( nBuf>3 && 0==memcmp("ive", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ |
+ *pnBuf = nBuf - 3; |
+ } |
+ } |
+ break; |
+ |
+ case 'z': |
+ if( nBuf>3 && 0==memcmp("ize", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ |
+ *pnBuf = nBuf - 3; |
+ } |
+ } |
+ break; |
+ |
+ } |
+ return ret; |
+} |
+ |
+ |
+static int fts5PorterStep1B2(char *aBuf, int *pnBuf){ |
+ int ret = 0; |
+ int nBuf = *pnBuf; |
+ switch( aBuf[nBuf-2] ){ |
+ |
+ case 'a': |
+ if( nBuf>2 && 0==memcmp("at", &aBuf[nBuf-2], 2) ){ |
+ memcpy(&aBuf[nBuf-2], "ate", 3); |
+ *pnBuf = nBuf - 2 + 3; |
+ ret = 1; |
+ } |
+ break; |
+ |
+ case 'b': |
+ if( nBuf>2 && 0==memcmp("bl", &aBuf[nBuf-2], 2) ){ |
+ memcpy(&aBuf[nBuf-2], "ble", 3); |
+ *pnBuf = nBuf - 2 + 3; |
+ ret = 1; |
+ } |
+ break; |
+ |
+ case 'i': |
+ if( nBuf>2 && 0==memcmp("iz", &aBuf[nBuf-2], 2) ){ |
+ memcpy(&aBuf[nBuf-2], "ize", 3); |
+ *pnBuf = nBuf - 2 + 3; |
+ ret = 1; |
+ } |
+ break; |
+ |
+ } |
+ return ret; |
+} |
+ |
+ |
+static int fts5PorterStep2(char *aBuf, int *pnBuf){ |
+ int ret = 0; |
+ int nBuf = *pnBuf; |
+ switch( aBuf[nBuf-2] ){ |
+ |
+ case 'a': |
+ if( nBuf>7 && 0==memcmp("ational", &aBuf[nBuf-7], 7) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-7) ){ |
+ memcpy(&aBuf[nBuf-7], "ate", 3); |
+ *pnBuf = nBuf - 7 + 3; |
+ } |
+ }else if( nBuf>6 && 0==memcmp("tional", &aBuf[nBuf-6], 6) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-6) ){ |
+ memcpy(&aBuf[nBuf-6], "tion", 4); |
+ *pnBuf = nBuf - 6 + 4; |
+ } |
+ } |
+ break; |
+ |
+ case 'c': |
+ if( nBuf>4 && 0==memcmp("enci", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ |
+ memcpy(&aBuf[nBuf-4], "ence", 4); |
+ *pnBuf = nBuf - 4 + 4; |
+ } |
+ }else if( nBuf>4 && 0==memcmp("anci", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ |
+ memcpy(&aBuf[nBuf-4], "ance", 4); |
+ *pnBuf = nBuf - 4 + 4; |
+ } |
+ } |
+ break; |
+ |
+ case 'e': |
+ if( nBuf>4 && 0==memcmp("izer", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ |
+ memcpy(&aBuf[nBuf-4], "ize", 3); |
+ *pnBuf = nBuf - 4 + 3; |
+ } |
+ } |
+ break; |
+ |
+ case 'g': |
+ if( nBuf>4 && 0==memcmp("logi", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ |
+ memcpy(&aBuf[nBuf-4], "log", 3); |
+ *pnBuf = nBuf - 4 + 3; |
+ } |
+ } |
+ break; |
+ |
+ case 'l': |
+ if( nBuf>3 && 0==memcmp("bli", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-3) ){ |
+ memcpy(&aBuf[nBuf-3], "ble", 3); |
+ *pnBuf = nBuf - 3 + 3; |
+ } |
+ }else if( nBuf>4 && 0==memcmp("alli", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ |
+ memcpy(&aBuf[nBuf-4], "al", 2); |
+ *pnBuf = nBuf - 4 + 2; |
+ } |
+ }else if( nBuf>5 && 0==memcmp("entli", &aBuf[nBuf-5], 5) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ |
+ memcpy(&aBuf[nBuf-5], "ent", 3); |
+ *pnBuf = nBuf - 5 + 3; |
+ } |
+ }else if( nBuf>3 && 0==memcmp("eli", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-3) ){ |
+ memcpy(&aBuf[nBuf-3], "e", 1); |
+ *pnBuf = nBuf - 3 + 1; |
+ } |
+ }else if( nBuf>5 && 0==memcmp("ousli", &aBuf[nBuf-5], 5) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ |
+ memcpy(&aBuf[nBuf-5], "ous", 3); |
+ *pnBuf = nBuf - 5 + 3; |
+ } |
+ } |
+ break; |
+ |
+ case 'o': |
+ if( nBuf>7 && 0==memcmp("ization", &aBuf[nBuf-7], 7) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-7) ){ |
+ memcpy(&aBuf[nBuf-7], "ize", 3); |
+ *pnBuf = nBuf - 7 + 3; |
+ } |
+ }else if( nBuf>5 && 0==memcmp("ation", &aBuf[nBuf-5], 5) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ |
+ memcpy(&aBuf[nBuf-5], "ate", 3); |
+ *pnBuf = nBuf - 5 + 3; |
+ } |
+ }else if( nBuf>4 && 0==memcmp("ator", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ |
+ memcpy(&aBuf[nBuf-4], "ate", 3); |
+ *pnBuf = nBuf - 4 + 3; |
+ } |
+ } |
+ break; |
+ |
+ case 's': |
+ if( nBuf>5 && 0==memcmp("alism", &aBuf[nBuf-5], 5) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ |
+ memcpy(&aBuf[nBuf-5], "al", 2); |
+ *pnBuf = nBuf - 5 + 2; |
+ } |
+ }else if( nBuf>7 && 0==memcmp("iveness", &aBuf[nBuf-7], 7) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-7) ){ |
+ memcpy(&aBuf[nBuf-7], "ive", 3); |
+ *pnBuf = nBuf - 7 + 3; |
+ } |
+ }else if( nBuf>7 && 0==memcmp("fulness", &aBuf[nBuf-7], 7) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-7) ){ |
+ memcpy(&aBuf[nBuf-7], "ful", 3); |
+ *pnBuf = nBuf - 7 + 3; |
+ } |
+ }else if( nBuf>7 && 0==memcmp("ousness", &aBuf[nBuf-7], 7) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-7) ){ |
+ memcpy(&aBuf[nBuf-7], "ous", 3); |
+ *pnBuf = nBuf - 7 + 3; |
+ } |
+ } |
+ break; |
+ |
+ case 't': |
+ if( nBuf>5 && 0==memcmp("aliti", &aBuf[nBuf-5], 5) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ |
+ memcpy(&aBuf[nBuf-5], "al", 2); |
+ *pnBuf = nBuf - 5 + 2; |
+ } |
+ }else if( nBuf>5 && 0==memcmp("iviti", &aBuf[nBuf-5], 5) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ |
+ memcpy(&aBuf[nBuf-5], "ive", 3); |
+ *pnBuf = nBuf - 5 + 3; |
+ } |
+ }else if( nBuf>6 && 0==memcmp("biliti", &aBuf[nBuf-6], 6) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-6) ){ |
+ memcpy(&aBuf[nBuf-6], "ble", 3); |
+ *pnBuf = nBuf - 6 + 3; |
+ } |
+ } |
+ break; |
+ |
+ } |
+ return ret; |
+} |
+ |
+ |
+static int fts5PorterStep3(char *aBuf, int *pnBuf){ |
+ int ret = 0; |
+ int nBuf = *pnBuf; |
+ switch( aBuf[nBuf-2] ){ |
+ |
+ case 'a': |
+ if( nBuf>4 && 0==memcmp("ical", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ |
+ memcpy(&aBuf[nBuf-4], "ic", 2); |
+ *pnBuf = nBuf - 4 + 2; |
+ } |
+ } |
+ break; |
+ |
+ case 's': |
+ if( nBuf>4 && 0==memcmp("ness", &aBuf[nBuf-4], 4) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ |
+ *pnBuf = nBuf - 4; |
+ } |
+ } |
+ break; |
+ |
+ case 't': |
+ if( nBuf>5 && 0==memcmp("icate", &aBuf[nBuf-5], 5) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ |
+ memcpy(&aBuf[nBuf-5], "ic", 2); |
+ *pnBuf = nBuf - 5 + 2; |
+ } |
+ }else if( nBuf>5 && 0==memcmp("iciti", &aBuf[nBuf-5], 5) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ |
+ memcpy(&aBuf[nBuf-5], "ic", 2); |
+ *pnBuf = nBuf - 5 + 2; |
+ } |
+ } |
+ break; |
+ |
+ case 'u': |
+ if( nBuf>3 && 0==memcmp("ful", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-3) ){ |
+ *pnBuf = nBuf - 3; |
+ } |
+ } |
+ break; |
+ |
+ case 'v': |
+ if( nBuf>5 && 0==memcmp("ative", &aBuf[nBuf-5], 5) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ |
+ *pnBuf = nBuf - 5; |
+ } |
+ } |
+ break; |
+ |
+ case 'z': |
+ if( nBuf>5 && 0==memcmp("alize", &aBuf[nBuf-5], 5) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ |
+ memcpy(&aBuf[nBuf-5], "al", 2); |
+ *pnBuf = nBuf - 5 + 2; |
+ } |
+ } |
+ break; |
+ |
+ } |
+ return ret; |
+} |
+ |
+ |
+static int fts5PorterStep1B(char *aBuf, int *pnBuf){ |
+ int ret = 0; |
+ int nBuf = *pnBuf; |
+ switch( aBuf[nBuf-2] ){ |
+ |
+ case 'e': |
+ if( nBuf>3 && 0==memcmp("eed", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_MGt0(aBuf, nBuf-3) ){ |
+ memcpy(&aBuf[nBuf-3], "ee", 2); |
+ *pnBuf = nBuf - 3 + 2; |
+ } |
+ }else if( nBuf>2 && 0==memcmp("ed", &aBuf[nBuf-2], 2) ){ |
+ if( fts5Porter_Vowel(aBuf, nBuf-2) ){ |
+ *pnBuf = nBuf - 2; |
+ ret = 1; |
+ } |
+ } |
+ break; |
+ |
+ case 'n': |
+ if( nBuf>3 && 0==memcmp("ing", &aBuf[nBuf-3], 3) ){ |
+ if( fts5Porter_Vowel(aBuf, nBuf-3) ){ |
+ *pnBuf = nBuf - 3; |
+ ret = 1; |
+ } |
+ } |
+ break; |
+ |
+ } |
+ return ret; |
+} |
+ |
+/* |
+** GENERATED CODE ENDS HERE (mkportersteps.tcl) |
+*************************************************************************** |
+**************************************************************************/ |
+ |
+static void fts5PorterStep1A(char *aBuf, int *pnBuf){ |
+ int nBuf = *pnBuf; |
+ if( aBuf[nBuf-1]=='s' ){ |
+ if( aBuf[nBuf-2]=='e' ){ |
+ if( (nBuf>4 && aBuf[nBuf-4]=='s' && aBuf[nBuf-3]=='s') |
+ || (nBuf>3 && aBuf[nBuf-3]=='i' ) |
+ ){ |
+ *pnBuf = nBuf-2; |
+ }else{ |
+ *pnBuf = nBuf-1; |
+ } |
+ } |
+ else if( aBuf[nBuf-2]!='s' ){ |
+ *pnBuf = nBuf-1; |
+ } |
+ } |
+} |
+ |
+static int fts5PorterCb( |
+ void *pCtx, |
+ int tflags, |
+ const char *pToken, |
+ int nToken, |
+ int iStart, |
+ int iEnd |
+){ |
+ PorterContext *p = (PorterContext*)pCtx; |
+ |
+ char *aBuf; |
+ int nBuf; |
+ |
+ if( nToken>FTS5_PORTER_MAX_TOKEN || nToken<3 ) goto pass_through; |
+ aBuf = p->aBuf; |
+ nBuf = nToken; |
+ memcpy(aBuf, pToken, nBuf); |
+ |
+ /* Step 1. */ |
+ fts5PorterStep1A(aBuf, &nBuf); |
+ if( fts5PorterStep1B(aBuf, &nBuf) ){ |
+ if( fts5PorterStep1B2(aBuf, &nBuf)==0 ){ |
+ char c = aBuf[nBuf-1]; |
+ if( fts5PorterIsVowel(c, 0)==0 |
+ && c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2] |
+ ){ |
+ nBuf--; |
+ }else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){ |
+ aBuf[nBuf++] = 'e'; |
+ } |
+ } |
+ } |
+ |
+ /* Step 1C. */ |
+ if( aBuf[nBuf-1]=='y' && fts5Porter_Vowel(aBuf, nBuf-1) ){ |
+ aBuf[nBuf-1] = 'i'; |
+ } |
+ |
+ /* Steps 2 through 4. */ |
+ fts5PorterStep2(aBuf, &nBuf); |
+ fts5PorterStep3(aBuf, &nBuf); |
+ fts5PorterStep4(aBuf, &nBuf); |
+ |
+ /* Step 5a. */ |
+ assert( nBuf>0 ); |
+ if( aBuf[nBuf-1]=='e' ){ |
+ if( fts5Porter_MGt1(aBuf, nBuf-1) |
+ || (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1)) |
+ ){ |
+ nBuf--; |
+ } |
+ } |
+ |
+ /* Step 5b. */ |
+ if( nBuf>1 && aBuf[nBuf-1]=='l' |
+ && aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1) |
+ ){ |
+ nBuf--; |
+ } |
+ |
+ return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd); |
+ |
+ pass_through: |
+ return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); |
+} |
+ |
+/* |
+** Tokenize using the porter tokenizer. |
+*/ |
+static int fts5PorterTokenize( |
+ Fts5Tokenizer *pTokenizer, |
+ void *pCtx, |
+ int flags, |
+ const char *pText, int nText, |
+ int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) |
+){ |
+ PorterTokenizer *p = (PorterTokenizer*)pTokenizer; |
+ PorterContext sCtx; |
+ sCtx.xToken = xToken; |
+ sCtx.pCtx = pCtx; |
+ sCtx.aBuf = p->aBuf; |
+ return p->tokenizer.xTokenize( |
+ p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb |
+ ); |
+} |
+ |
+/* |
+** Register all built-in tokenizers with FTS5. |
+*/ |
+static int sqlite3Fts5TokenizerInit(fts5_api *pApi){ |
+ struct BuiltinTokenizer { |
+ const char *zName; |
+ fts5_tokenizer x; |
+ } aBuiltin[] = { |
+ { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}}, |
+ { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }}, |
+ { "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }}, |
+ }; |
+ |
+ int rc = SQLITE_OK; /* Return code */ |
+ int i; /* To iterate through builtin functions */ |
+ |
+ for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){ |
+ rc = pApi->xCreateTokenizer(pApi, |
+ aBuiltin[i].zName, |
+ (void*)pApi, |
+ &aBuiltin[i].x, |
+ 0 |
+ ); |
+ } |
+ |
+ return rc; |
+} |
+ |
+ |
+ |
+/* |
+** 2012 May 25 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+*/ |
+ |
+/* |
+** DO NOT EDIT THIS MACHINE GENERATED FILE. |
+*/ |
+ |
+ |
+/* #include <assert.h> */ |
+ |
+/* |
+** Return true if the argument corresponds to a unicode codepoint |
+** classified as either a letter or a number. Otherwise false. |
+** |
+** The results are undefined if the value passed to this function |
+** is less than zero. |
+*/ |
+static int sqlite3Fts5UnicodeIsalnum(int c){ |
+ /* Each unsigned integer in the following array corresponds to a contiguous |
+ ** range of unicode codepoints that are not either letters or numbers (i.e. |
+ ** codepoints for which this function should return 0). |
+ ** |
+ ** The most significant 22 bits in each 32-bit value contain the first |
+ ** codepoint in the range. The least significant 10 bits are used to store |
+ ** the size of the range (always at least 1). In other words, the value |
+ ** ((C<<22) + N) represents a range of N codepoints starting with codepoint |
+ ** C. It is not possible to represent a range larger than 1023 codepoints |
+ ** using this format. |
+ */ |
+ static const unsigned int aEntry[] = { |
+ 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, |
+ 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, |
+ 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, |
+ 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, |
+ 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, |
+ 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, |
+ 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, |
+ 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, |
+ 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, |
+ 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, |
+ 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, |
+ 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, |
+ 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, |
+ 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, |
+ 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, |
+ 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, |
+ 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, |
+ 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, |
+ 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, |
+ 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, |
+ 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, |
+ 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, |
+ 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, |
+ 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, |
+ 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, |
+ 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, |
+ 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, |
+ 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, |
+ 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, |
+ 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, |
+ 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, |
+ 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, |
+ 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, |
+ 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, |
+ 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, |
+ 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, |
+ 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, |
+ 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, |
+ 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, |
+ 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, |
+ 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, |
+ 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, |
+ 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, |
+ 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, |
+ 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, |
+ 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, |
+ 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, |
+ 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, |
+ 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, |
+ 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, |
+ 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, |
+ 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, |
+ 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, |
+ 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, |
+ 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, |
+ 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, |
+ 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, |
+ 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, |
+ 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, |
+ 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, |
+ 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, |
+ 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, |
+ 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, |
+ 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, |
+ 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, |
+ 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, |
+ 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, |
+ 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, |
+ 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, |
+ 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, |
+ 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, |
+ 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, |
+ 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, |
+ 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, |
+ 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, |
+ 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, |
+ 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, |
+ 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, |
+ 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, |
+ 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, |
+ 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, |
+ 0x380400F0, |
+ }; |
+ static const unsigned int aAscii[4] = { |
+ 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, |
+ }; |
+ |
+ if( (unsigned int)c<128 ){ |
+ return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); |
+ }else if( (unsigned int)c<(1<<22) ){ |
+ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; |
+ int iRes = 0; |
+ int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; |
+ int iLo = 0; |
+ while( iHi>=iLo ){ |
+ int iTest = (iHi + iLo) / 2; |
+ if( key >= aEntry[iTest] ){ |
+ iRes = iTest; |
+ iLo = iTest+1; |
+ }else{ |
+ iHi = iTest-1; |
+ } |
+ } |
+ assert( aEntry[0]<key ); |
+ assert( key>=aEntry[iRes] ); |
+ return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); |
+ } |
+ return 1; |
+} |
+ |
+ |
+/* |
+** If the argument is a codepoint corresponding to a lowercase letter |
+** in the ASCII range with a diacritic added, return the codepoint |
+** of the ASCII letter only. For example, if passed 235 - "LATIN |
+** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER |
+** E"). The resuls of passing a codepoint that corresponds to an |
+** uppercase letter are undefined. |
+*/ |
+static int fts5_remove_diacritic(int c){ |
+ unsigned short aDia[] = { |
+ 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, |
+ 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, |
+ 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, |
+ 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, |
+ 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, |
+ 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, |
+ 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, |
+ 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, |
+ 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, |
+ 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, |
+ 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, |
+ 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, |
+ 62924, 63050, 63082, 63274, 63390, |
+ }; |
+ char aChar[] = { |
+ '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', |
+ 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', |
+ 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', |
+ 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', |
+ 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', |
+ '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', |
+ 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', |
+ 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', |
+ 'e', 'i', 'o', 'u', 'y', |
+ }; |
+ |
+ unsigned int key = (((unsigned int)c)<<3) | 0x00000007; |
+ int iRes = 0; |
+ int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; |
+ int iLo = 0; |
+ while( iHi>=iLo ){ |
+ int iTest = (iHi + iLo) / 2; |
+ if( key >= aDia[iTest] ){ |
+ iRes = iTest; |
+ iLo = iTest+1; |
+ }else{ |
+ iHi = iTest-1; |
+ } |
+ } |
+ assert( key>=aDia[iRes] ); |
+ return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); |
+} |
+ |
+ |
+/* |
+** Return true if the argument interpreted as a unicode codepoint |
+** is a diacritical modifier character. |
+*/ |
+static int sqlite3Fts5UnicodeIsdiacritic(int c){ |
+ unsigned int mask0 = 0x08029FDF; |
+ unsigned int mask1 = 0x000361F8; |
+ if( c<768 || c>817 ) return 0; |
+ return (c < 768+32) ? |
+ (mask0 & (1 << (c-768))) : |
+ (mask1 & (1 << (c-768-32))); |
+} |
+ |
+ |
+/* |
+** Interpret the argument as a unicode codepoint. If the codepoint |
+** is an upper case character that has a lower case equivalent, |
+** return the codepoint corresponding to the lower case version. |
+** Otherwise, return a copy of the argument. |
+** |
+** The results are undefined if the value passed to this function |
+** is less than zero. |
+*/ |
+static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ |
+ /* Each entry in the following array defines a rule for folding a range |
+ ** of codepoints to lower case. The rule applies to a range of nRange |
+ ** codepoints starting at codepoint iCode. |
+ ** |
+ ** If the least significant bit in flags is clear, then the rule applies |
+ ** to all nRange codepoints (i.e. all nRange codepoints are upper case and |
+ ** need to be folded). Or, if it is set, then the rule only applies to |
+ ** every second codepoint in the range, starting with codepoint C. |
+ ** |
+ ** The 7 most significant bits in flags are an index into the aiOff[] |
+ ** array. If a specific codepoint C does require folding, then its lower |
+ ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). |
+ ** |
+ ** The contents of this array are generated by parsing the CaseFolding.txt |
+ ** file distributed as part of the "Unicode Character Database". See |
+ ** http://www.unicode.org for details. |
+ */ |
+ static const struct TableEntry { |
+ unsigned short iCode; |
+ unsigned char flags; |
+ unsigned char nRange; |
+ } aEntry[] = { |
+ {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, |
+ {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, |
+ {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, |
+ {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, |
+ {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, |
+ {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, |
+ {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, |
+ {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, |
+ {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, |
+ {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, |
+ {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, |
+ {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, |
+ {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, |
+ {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, |
+ {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, |
+ {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, |
+ {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, |
+ {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, |
+ {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, |
+ {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, |
+ {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, |
+ {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, |
+ {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, |
+ {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, |
+ {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, |
+ {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, |
+ {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, |
+ {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, |
+ {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, |
+ {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, |
+ {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, |
+ {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, |
+ {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, |
+ {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, |
+ {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, |
+ {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, |
+ {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, |
+ {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, |
+ {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, |
+ {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, |
+ {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, |
+ {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, |
+ {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, |
+ {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, |
+ {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, |
+ {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, |
+ {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, |
+ {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, |
+ {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, |
+ {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, |
+ {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, |
+ {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, |
+ {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, |
+ {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, |
+ {65313, 14, 26}, |
+ }; |
+ static const unsigned short aiOff[] = { |
+ 1, 2, 8, 15, 16, 26, 28, 32, |
+ 37, 38, 40, 48, 63, 64, 69, 71, |
+ 79, 80, 116, 202, 203, 205, 206, 207, |
+ 209, 210, 211, 213, 214, 217, 218, 219, |
+ 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, |
+ 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, |
+ 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, |
+ 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, |
+ 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, |
+ 65514, 65521, 65527, 65528, 65529, |
+ }; |
+ |
+ int ret = c; |
+ |
+ assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); |
+ |
+ if( c<128 ){ |
+ if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); |
+ }else if( c<65536 ){ |
+ const struct TableEntry *p; |
+ int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; |
+ int iLo = 0; |
+ int iRes = -1; |
+ |
+ assert( c>aEntry[0].iCode ); |
+ while( iHi>=iLo ){ |
+ int iTest = (iHi + iLo) / 2; |
+ int cmp = (c - aEntry[iTest].iCode); |
+ if( cmp>=0 ){ |
+ iRes = iTest; |
+ iLo = iTest+1; |
+ }else{ |
+ iHi = iTest-1; |
+ } |
+ } |
+ |
+ assert( iRes>=0 && c>=aEntry[iRes].iCode ); |
+ p = &aEntry[iRes]; |
+ if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ |
+ ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; |
+ assert( ret>0 ); |
+ } |
+ |
+ if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret); |
+ } |
+ |
+ else if( c>=66560 && c<66600 ){ |
+ ret = c + 40; |
+ } |
+ |
+ return ret; |
+} |
+ |
+/* |
+** 2015 May 30 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** Routines for varint serialization and deserialization. |
+*/ |
+ |
+ |
+/* #include "fts5Int.h" */ |
+ |
+/* |
+** This is a copy of the sqlite3GetVarint32() routine from the SQLite core. |
+** Except, this version does handle the single byte case that the core |
+** version depends on being handled before its function is called. |
+*/ |
+static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){ |
+ u32 a,b; |
+ |
+ /* The 1-byte case. Overwhelmingly the most common. */ |
+ a = *p; |
+ /* a: p0 (unmasked) */ |
+ if (!(a&0x80)) |
+ { |
+ /* Values between 0 and 127 */ |
+ *v = a; |
+ return 1; |
+ } |
+ |
+ /* The 2-byte case */ |
+ p++; |
+ b = *p; |
+ /* b: p1 (unmasked) */ |
+ if (!(b&0x80)) |
+ { |
+ /* Values between 128 and 16383 */ |
+ a &= 0x7f; |
+ a = a<<7; |
+ *v = a | b; |
+ return 2; |
+ } |
+ |
+ /* The 3-byte case */ |
+ p++; |
+ a = a<<14; |
+ a |= *p; |
+ /* a: p0<<14 | p2 (unmasked) */ |
+ if (!(a&0x80)) |
+ { |
+ /* Values between 16384 and 2097151 */ |
+ a &= (0x7f<<14)|(0x7f); |
+ b &= 0x7f; |
+ b = b<<7; |
+ *v = a | b; |
+ return 3; |
+ } |
+ |
+ /* A 32-bit varint is used to store size information in btrees. |
+ ** Objects are rarely larger than 2MiB limit of a 3-byte varint. |
+ ** A 3-byte varint is sufficient, for example, to record the size |
+ ** of a 1048569-byte BLOB or string. |
+ ** |
+ ** We only unroll the first 1-, 2-, and 3- byte cases. The very |
+ ** rare larger cases can be handled by the slower 64-bit varint |
+ ** routine. |
+ */ |
+ { |
+ u64 v64; |
+ u8 n; |
+ p -= 2; |
+ n = sqlite3Fts5GetVarint(p, &v64); |
+ *v = (u32)v64; |
+ assert( n>3 && n<=9 ); |
+ return n; |
+ } |
+} |
+ |
+ |
+/* |
+** Bitmasks used by sqlite3GetVarint(). These precomputed constants |
+** are defined here rather than simply putting the constant expressions |
+** inline in order to work around bugs in the RVT compiler. |
+** |
+** SLOT_2_0 A mask for (0x7f<<14) | 0x7f |
+** |
+** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0 |
+*/ |
+#define SLOT_2_0 0x001fc07f |
+#define SLOT_4_2_0 0xf01fc07f |
+ |
+/* |
+** Read a 64-bit variable-length integer from memory starting at p[0]. |
+** Return the number of bytes read. The value is stored in *v. |
+*/ |
+static u8 sqlite3Fts5GetVarint(const unsigned char *p, u64 *v){ |
+ u32 a,b,s; |
+ |
+ a = *p; |
+ /* a: p0 (unmasked) */ |
+ if (!(a&0x80)) |
+ { |
+ *v = a; |
+ return 1; |
+ } |
+ |
+ p++; |
+ b = *p; |
+ /* b: p1 (unmasked) */ |
+ if (!(b&0x80)) |
+ { |
+ a &= 0x7f; |
+ a = a<<7; |
+ a |= b; |
+ *v = a; |
+ return 2; |
+ } |
+ |
+ /* Verify that constants are precomputed correctly */ |
+ assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); |
+ assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); |
+ |
+ p++; |
+ a = a<<14; |
+ a |= *p; |
+ /* a: p0<<14 | p2 (unmasked) */ |
+ if (!(a&0x80)) |
+ { |
+ a &= SLOT_2_0; |
+ b &= 0x7f; |
+ b = b<<7; |
+ a |= b; |
+ *v = a; |
+ return 3; |
+ } |
+ |
+ /* CSE1 from below */ |
+ a &= SLOT_2_0; |
+ p++; |
+ b = b<<14; |
+ b |= *p; |
+ /* b: p1<<14 | p3 (unmasked) */ |
+ if (!(b&0x80)) |
+ { |
+ b &= SLOT_2_0; |
+ /* moved CSE1 up */ |
+ /* a &= (0x7f<<14)|(0x7f); */ |
+ a = a<<7; |
+ a |= b; |
+ *v = a; |
+ return 4; |
+ } |
+ |
+ /* a: p0<<14 | p2 (masked) */ |
+ /* b: p1<<14 | p3 (unmasked) */ |
+ /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ |
+ /* moved CSE1 up */ |
+ /* a &= (0x7f<<14)|(0x7f); */ |
+ b &= SLOT_2_0; |
+ s = a; |
+ /* s: p0<<14 | p2 (masked) */ |
+ |
+ p++; |
+ a = a<<14; |
+ a |= *p; |
+ /* a: p0<<28 | p2<<14 | p4 (unmasked) */ |
+ if (!(a&0x80)) |
+ { |
+ /* we can skip these cause they were (effectively) done above in calc'ing s */ |
+ /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ |
+ /* b &= (0x7f<<14)|(0x7f); */ |
+ b = b<<7; |
+ a |= b; |
+ s = s>>18; |
+ *v = ((u64)s)<<32 | a; |
+ return 5; |
+ } |
+ |
+ /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ |
+ s = s<<7; |
+ s |= b; |
+ /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ |
+ |
+ p++; |
+ b = b<<14; |
+ b |= *p; |
+ /* b: p1<<28 | p3<<14 | p5 (unmasked) */ |
+ if (!(b&0x80)) |
+ { |
+ /* we can skip this cause it was (effectively) done above in calc'ing s */ |
+ /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ |
+ a &= SLOT_2_0; |
+ a = a<<7; |
+ a |= b; |
+ s = s>>18; |
+ *v = ((u64)s)<<32 | a; |
+ return 6; |
+ } |
+ |
+ p++; |
+ a = a<<14; |
+ a |= *p; |
+ /* a: p2<<28 | p4<<14 | p6 (unmasked) */ |
+ if (!(a&0x80)) |
+ { |
+ a &= SLOT_4_2_0; |
+ b &= SLOT_2_0; |
+ b = b<<7; |
+ a |= b; |
+ s = s>>11; |
+ *v = ((u64)s)<<32 | a; |
+ return 7; |
+ } |
+ |
+ /* CSE2 from below */ |
+ a &= SLOT_2_0; |
+ p++; |
+ b = b<<14; |
+ b |= *p; |
+ /* b: p3<<28 | p5<<14 | p7 (unmasked) */ |
+ if (!(b&0x80)) |
+ { |
+ b &= SLOT_4_2_0; |
+ /* moved CSE2 up */ |
+ /* a &= (0x7f<<14)|(0x7f); */ |
+ a = a<<7; |
+ a |= b; |
+ s = s>>4; |
+ *v = ((u64)s)<<32 | a; |
+ return 8; |
+ } |
+ |
+ p++; |
+ a = a<<15; |
+ a |= *p; |
+ /* a: p4<<29 | p6<<15 | p8 (unmasked) */ |
+ |
+ /* moved CSE2 up */ |
+ /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */ |
+ b &= SLOT_2_0; |
+ b = b<<8; |
+ a |= b; |
+ |
+ s = s<<4; |
+ b = p[-4]; |
+ b &= 0x7f; |
+ b = b>>3; |
+ s |= b; |
+ |
+ *v = ((u64)s)<<32 | a; |
+ |
+ return 9; |
+} |
+ |
+/* |
+** The variable-length integer encoding is as follows: |
+** |
+** KEY: |
+** A = 0xxxxxxx 7 bits of data and one flag bit |
+** B = 1xxxxxxx 7 bits of data and one flag bit |
+** C = xxxxxxxx 8 bits of data |
+** |
+** 7 bits - A |
+** 14 bits - BA |
+** 21 bits - BBA |
+** 28 bits - BBBA |
+** 35 bits - BBBBA |
+** 42 bits - BBBBBA |
+** 49 bits - BBBBBBA |
+** 56 bits - BBBBBBBA |
+** 64 bits - BBBBBBBBC |
+*/ |
+ |
+#ifdef SQLITE_NOINLINE |
+# define FTS5_NOINLINE SQLITE_NOINLINE |
+#else |
+# define FTS5_NOINLINE |
+#endif |
+ |
+/* |
+** Write a 64-bit variable-length integer to memory starting at p[0]. |
+** The length of data write will be between 1 and 9 bytes. The number |
+** of bytes written is returned. |
+** |
+** A variable-length integer consists of the lower 7 bits of each byte |
+** for all bytes that have the 8th bit set and one byte with the 8th |
+** bit clear. Except, if we get to the 9th byte, it stores the full |
+** 8 bits and is the last byte. |
+*/ |
+static int FTS5_NOINLINE fts5PutVarint64(unsigned char *p, u64 v){ |
+ int i, j, n; |
+ u8 buf[10]; |
+ if( v & (((u64)0xff000000)<<32) ){ |
+ p[8] = (u8)v; |
+ v >>= 8; |
+ for(i=7; i>=0; i--){ |
+ p[i] = (u8)((v & 0x7f) | 0x80); |
+ v >>= 7; |
+ } |
+ return 9; |
+ } |
+ n = 0; |
+ do{ |
+ buf[n++] = (u8)((v & 0x7f) | 0x80); |
+ v >>= 7; |
+ }while( v!=0 ); |
+ buf[0] &= 0x7f; |
+ assert( n<=9 ); |
+ for(i=0, j=n-1; j>=0; j--, i++){ |
+ p[i] = buf[j]; |
+ } |
+ return n; |
+} |
+ |
+static int sqlite3Fts5PutVarint(unsigned char *p, u64 v){ |
+ if( v<=0x7f ){ |
+ p[0] = v&0x7f; |
+ return 1; |
+ } |
+ if( v<=0x3fff ){ |
+ p[0] = ((v>>7)&0x7f)|0x80; |
+ p[1] = v&0x7f; |
+ return 2; |
+ } |
+ return fts5PutVarint64(p,v); |
+} |
+ |
+ |
+static int sqlite3Fts5GetVarintLen(u32 iVal){ |
+#if 0 |
+ if( iVal<(1 << 7 ) ) return 1; |
+#endif |
+ assert( iVal>=(1 << 7) ); |
+ if( iVal<(1 << 14) ) return 2; |
+ if( iVal<(1 << 21) ) return 3; |
+ if( iVal<(1 << 28) ) return 4; |
+ return 5; |
+} |
+ |
+ |
+/* |
+** 2015 May 08 |
+** |
+** The author disclaims copyright to this source code. In place of |
+** a legal notice, here is a blessing: |
+** |
+** May you do good and not evil. |
+** May you find forgiveness for yourself and forgive others. |
+** May you share freely, never taking more than you give. |
+** |
+****************************************************************************** |
+** |
+** This is an SQLite virtual table module implementing direct access to an |
+** existing FTS5 index. The module may create several different types of |
+** tables: |
+** |
+** col: |
+** CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col)); |
+** |
+** One row for each term/column combination. The value of $doc is set to |
+** the number of fts5 rows that contain at least one instance of term |
+** $term within column $col. Field $cnt is set to the total number of |
+** instances of term $term in column $col (in any row of the fts5 table). |
+** |
+** row: |
+** CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term)); |
+** |
+** One row for each term in the database. The value of $doc is set to |
+** the number of fts5 rows that contain at least one instance of term |
+** $term. Field $cnt is set to the total number of instances of term |
+** $term in the database. |
+*/ |
+ |
+ |
+/* #include "fts5Int.h" */ |
+ |
+ |
+typedef struct Fts5VocabTable Fts5VocabTable; |
+typedef struct Fts5VocabCursor Fts5VocabCursor; |
+ |
+struct Fts5VocabTable { |
+ sqlite3_vtab base; |
+ char *zFts5Tbl; /* Name of fts5 table */ |
+ char *zFts5Db; /* Db containing fts5 table */ |
+ sqlite3 *db; /* Database handle */ |
+ Fts5Global *pGlobal; /* FTS5 global object for this database */ |
+ int eType; /* FTS5_VOCAB_COL or ROW */ |
+}; |
+ |
+struct Fts5VocabCursor { |
+ sqlite3_vtab_cursor base; |
+ sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */ |
+ Fts5Index *pIndex; /* Associated FTS5 index */ |
+ |
+ int bEof; /* True if this cursor is at EOF */ |
+ Fts5IndexIter *pIter; /* Term/rowid iterator object */ |
+ |
+ int nLeTerm; /* Size of zLeTerm in bytes */ |
+ char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */ |
+ |
+ /* These are used by 'col' tables only */ |
+ Fts5Config *pConfig; /* Fts5 table configuration */ |
+ int iCol; |
+ i64 *aCnt; |
+ i64 *aDoc; |
+ |
+ /* Output values used by 'row' and 'col' tables */ |
+ i64 rowid; /* This table's current rowid value */ |
+ Fts5Buffer term; /* Current value of 'term' column */ |
+}; |
+ |
+#define FTS5_VOCAB_COL 0 |
+#define FTS5_VOCAB_ROW 1 |
+ |
+#define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt" |
+#define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt" |
+ |
+/* |
+** Bits for the mask used as the idxNum value by xBestIndex/xFilter. |
+*/ |
+#define FTS5_VOCAB_TERM_EQ 0x01 |
+#define FTS5_VOCAB_TERM_GE 0x02 |
+#define FTS5_VOCAB_TERM_LE 0x04 |
+ |
+ |
+/* |
+** Translate a string containing an fts5vocab table type to an |
+** FTS5_VOCAB_XXX constant. If successful, set *peType to the output |
+** value and return SQLITE_OK. Otherwise, set *pzErr to an error message |
+** and return SQLITE_ERROR. |
+*/ |
+static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){ |
+ int rc = SQLITE_OK; |
+ char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1); |
+ if( rc==SQLITE_OK ){ |
+ sqlite3Fts5Dequote(zCopy); |
+ if( sqlite3_stricmp(zCopy, "col")==0 ){ |
+ *peType = FTS5_VOCAB_COL; |
+ }else |
+ |
+ if( sqlite3_stricmp(zCopy, "row")==0 ){ |
+ *peType = FTS5_VOCAB_ROW; |
+ }else |
+ { |
+ *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy); |
+ rc = SQLITE_ERROR; |
+ } |
+ sqlite3_free(zCopy); |
+ } |
+ |
+ return rc; |
+} |
+ |
+ |
+/* |
+** The xDisconnect() virtual table method. |
+*/ |
+static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){ |
+ Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; |
+ sqlite3_free(pTab); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** The xDestroy() virtual table method. |
+*/ |
+static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){ |
+ Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; |
+ sqlite3_free(pTab); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** This function is the implementation of both the xConnect and xCreate |
+** methods of the FTS3 virtual table. |
+** |
+** The argv[] array contains the following: |
+** |
+** argv[0] -> module name ("fts5vocab") |
+** argv[1] -> database name |
+** argv[2] -> table name |
+** |
+** then: |
+** |
+** argv[3] -> name of fts5 table |
+** argv[4] -> type of fts5vocab table |
+** |
+** or, for tables in the TEMP schema only. |
+** |
+** argv[3] -> name of fts5 tables database |
+** argv[4] -> name of fts5 table |
+** argv[5] -> type of fts5vocab table |
+*/ |
+static int fts5VocabInitVtab( |
+ sqlite3 *db, /* The SQLite database connection */ |
+ void *pAux, /* Pointer to Fts5Global object */ |
+ int argc, /* Number of elements in argv array */ |
+ const char * const *argv, /* xCreate/xConnect argument array */ |
+ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ |
+ char **pzErr /* Write any error message here */ |
+){ |
+ const char *azSchema[] = { |
+ "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")", |
+ "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")" |
+ }; |
+ |
+ Fts5VocabTable *pRet = 0; |
+ int rc = SQLITE_OK; /* Return code */ |
+ int bDb; |
+ |
+ bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0); |
+ |
+ if( argc!=5 && bDb==0 ){ |
+ *pzErr = sqlite3_mprintf("wrong number of vtable arguments"); |
+ rc = SQLITE_ERROR; |
+ }else{ |
+ int nByte; /* Bytes of space to allocate */ |
+ const char *zDb = bDb ? argv[3] : argv[1]; |
+ const char *zTab = bDb ? argv[4] : argv[3]; |
+ const char *zType = bDb ? argv[5] : argv[4]; |
+ int nDb = (int)strlen(zDb)+1; |
+ int nTab = (int)strlen(zTab)+1; |
+ int eType = 0; |
+ |
+ rc = fts5VocabTableType(zType, pzErr, &eType); |
+ if( rc==SQLITE_OK ){ |
+ assert( eType>=0 && eType<ArraySize(azSchema) ); |
+ rc = sqlite3_declare_vtab(db, azSchema[eType]); |
+ } |
+ |
+ nByte = sizeof(Fts5VocabTable) + nDb + nTab; |
+ pRet = sqlite3Fts5MallocZero(&rc, nByte); |
+ if( pRet ){ |
+ pRet->pGlobal = (Fts5Global*)pAux; |
+ pRet->eType = eType; |
+ pRet->db = db; |
+ pRet->zFts5Tbl = (char*)&pRet[1]; |
+ pRet->zFts5Db = &pRet->zFts5Tbl[nTab]; |
+ memcpy(pRet->zFts5Tbl, zTab, nTab); |
+ memcpy(pRet->zFts5Db, zDb, nDb); |
+ sqlite3Fts5Dequote(pRet->zFts5Tbl); |
+ sqlite3Fts5Dequote(pRet->zFts5Db); |
+ } |
+ } |
+ |
+ *ppVTab = (sqlite3_vtab*)pRet; |
+ return rc; |
+} |
+ |
+ |
+/* |
+** The xConnect() and xCreate() methods for the virtual table. All the |
+** work is done in function fts5VocabInitVtab(). |
+*/ |
+static int fts5VocabConnectMethod( |
+ sqlite3 *db, /* Database connection */ |
+ void *pAux, /* Pointer to tokenizer hash table */ |
+ int argc, /* Number of elements in argv array */ |
+ const char * const *argv, /* xCreate/xConnect argument array */ |
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ |
+ char **pzErr /* OUT: sqlite3_malloc'd error message */ |
+){ |
+ return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); |
+} |
+static int fts5VocabCreateMethod( |
+ sqlite3 *db, /* Database connection */ |
+ void *pAux, /* Pointer to tokenizer hash table */ |
+ int argc, /* Number of elements in argv array */ |
+ const char * const *argv, /* xCreate/xConnect argument array */ |
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ |
+ char **pzErr /* OUT: sqlite3_malloc'd error message */ |
+){ |
+ return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); |
+} |
+ |
+/* |
+** Implementation of the xBestIndex method. |
+*/ |
+static int fts5VocabBestIndexMethod( |
+ sqlite3_vtab *pUnused, |
+ sqlite3_index_info *pInfo |
+){ |
+ int i; |
+ int iTermEq = -1; |
+ int iTermGe = -1; |
+ int iTermLe = -1; |
+ int idxNum = 0; |
+ int nArg = 0; |
+ |
+ UNUSED_PARAM(pUnused); |
+ |
+ for(i=0; i<pInfo->nConstraint; i++){ |
+ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; |
+ if( p->usable==0 ) continue; |
+ if( p->iColumn==0 ){ /* term column */ |
+ if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i; |
+ if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i; |
+ if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i; |
+ if( p->op==SQLITE_INDEX_CONSTRAINT_GE ) iTermGe = i; |
+ if( p->op==SQLITE_INDEX_CONSTRAINT_GT ) iTermGe = i; |
+ } |
+ } |
+ |
+ if( iTermEq>=0 ){ |
+ idxNum |= FTS5_VOCAB_TERM_EQ; |
+ pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg; |
+ pInfo->estimatedCost = 100; |
+ }else{ |
+ pInfo->estimatedCost = 1000000; |
+ if( iTermGe>=0 ){ |
+ idxNum |= FTS5_VOCAB_TERM_GE; |
+ pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg; |
+ pInfo->estimatedCost = pInfo->estimatedCost / 2; |
+ } |
+ if( iTermLe>=0 ){ |
+ idxNum |= FTS5_VOCAB_TERM_LE; |
+ pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg; |
+ pInfo->estimatedCost = pInfo->estimatedCost / 2; |
+ } |
+ } |
+ |
+ /* This virtual table always delivers results in ascending order of |
+ ** the "term" column (column 0). So if the user has requested this |
+ ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the |
+ ** sqlite3_index_info.orderByConsumed flag to tell the core the results |
+ ** are already in sorted order. */ |
+ if( pInfo->nOrderBy==1 |
+ && pInfo->aOrderBy[0].iColumn==0 |
+ && pInfo->aOrderBy[0].desc==0 |
+ ){ |
+ pInfo->orderByConsumed = 1; |
+ } |
+ |
+ pInfo->idxNum = idxNum; |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** Implementation of xOpen method. |
+*/ |
+static int fts5VocabOpenMethod( |
+ sqlite3_vtab *pVTab, |
+ sqlite3_vtab_cursor **ppCsr |
+){ |
+ Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab; |
+ Fts5Index *pIndex = 0; |
+ Fts5Config *pConfig = 0; |
+ Fts5VocabCursor *pCsr = 0; |
+ int rc = SQLITE_OK; |
+ sqlite3_stmt *pStmt = 0; |
+ char *zSql = 0; |
+ |
+ zSql = sqlite3Fts5Mprintf(&rc, |
+ "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'", |
+ pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl |
+ ); |
+ if( zSql ){ |
+ rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); |
+ } |
+ sqlite3_free(zSql); |
+ assert( rc==SQLITE_OK || pStmt==0 ); |
+ if( rc==SQLITE_ERROR ) rc = SQLITE_OK; |
+ |
+ if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ |
+ i64 iId = sqlite3_column_int64(pStmt, 0); |
+ pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &pConfig); |
+ } |
+ |
+ if( rc==SQLITE_OK && pIndex==0 ){ |
+ rc = sqlite3_finalize(pStmt); |
+ pStmt = 0; |
+ if( rc==SQLITE_OK ){ |
+ pVTab->zErrMsg = sqlite3_mprintf( |
+ "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl |
+ ); |
+ rc = SQLITE_ERROR; |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ int nByte = pConfig->nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor); |
+ pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte); |
+ } |
+ |
+ if( pCsr ){ |
+ pCsr->pIndex = pIndex; |
+ pCsr->pStmt = pStmt; |
+ pCsr->pConfig = pConfig; |
+ pCsr->aCnt = (i64*)&pCsr[1]; |
+ pCsr->aDoc = &pCsr->aCnt[pConfig->nCol]; |
+ }else{ |
+ sqlite3_finalize(pStmt); |
+ } |
+ |
+ *ppCsr = (sqlite3_vtab_cursor*)pCsr; |
+ return rc; |
+} |
+ |
+static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ |
+ pCsr->rowid = 0; |
+ sqlite3Fts5IterClose(pCsr->pIter); |
+ pCsr->pIter = 0; |
+ sqlite3_free(pCsr->zLeTerm); |
+ pCsr->nLeTerm = -1; |
+ pCsr->zLeTerm = 0; |
+} |
+ |
+/* |
+** Close the cursor. For additional information see the documentation |
+** on the xClose method of the virtual table interface. |
+*/ |
+static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; |
+ fts5VocabResetCursor(pCsr); |
+ sqlite3Fts5BufferFree(&pCsr->term); |
+ sqlite3_finalize(pCsr->pStmt); |
+ sqlite3_free(pCsr); |
+ return SQLITE_OK; |
+} |
+ |
+ |
+/* |
+** Advance the cursor to the next row in the table. |
+*/ |
+static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; |
+ Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; |
+ int rc = SQLITE_OK; |
+ int nCol = pCsr->pConfig->nCol; |
+ |
+ pCsr->rowid++; |
+ |
+ if( pTab->eType==FTS5_VOCAB_COL ){ |
+ for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){ |
+ if( pCsr->aDoc[pCsr->iCol] ) break; |
+ } |
+ } |
+ |
+ if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){ |
+ if( sqlite3Fts5IterEof(pCsr->pIter) ){ |
+ pCsr->bEof = 1; |
+ }else{ |
+ const char *zTerm; |
+ int nTerm; |
+ |
+ zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); |
+ if( pCsr->nLeTerm>=0 ){ |
+ int nCmp = MIN(nTerm, pCsr->nLeTerm); |
+ int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp); |
+ if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){ |
+ pCsr->bEof = 1; |
+ return SQLITE_OK; |
+ } |
+ } |
+ |
+ sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); |
+ memset(pCsr->aCnt, 0, nCol * sizeof(i64)); |
+ memset(pCsr->aDoc, 0, nCol * sizeof(i64)); |
+ pCsr->iCol = 0; |
+ |
+ assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); |
+ while( rc==SQLITE_OK ){ |
+ const u8 *pPos; int nPos; /* Position list */ |
+ i64 iPos = 0; /* 64-bit position read from poslist */ |
+ int iOff = 0; /* Current offset within position list */ |
+ |
+ pPos = pCsr->pIter->pData; |
+ nPos = pCsr->pIter->nData; |
+ switch( pCsr->pConfig->eDetail ){ |
+ case FTS5_DETAIL_FULL: |
+ pPos = pCsr->pIter->pData; |
+ nPos = pCsr->pIter->nData; |
+ if( pTab->eType==FTS5_VOCAB_ROW ){ |
+ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ |
+ pCsr->aCnt[0]++; |
+ } |
+ pCsr->aDoc[0]++; |
+ }else{ |
+ int iCol = -1; |
+ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ |
+ int ii = FTS5_POS2COLUMN(iPos); |
+ pCsr->aCnt[ii]++; |
+ if( iCol!=ii ){ |
+ if( ii>=nCol ){ |
+ rc = FTS5_CORRUPT; |
+ break; |
+ } |
+ pCsr->aDoc[ii]++; |
+ iCol = ii; |
+ } |
+ } |
+ } |
+ break; |
+ |
+ case FTS5_DETAIL_COLUMNS: |
+ if( pTab->eType==FTS5_VOCAB_ROW ){ |
+ pCsr->aDoc[0]++; |
+ }else{ |
+ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){ |
+ assert_nc( iPos>=0 && iPos<nCol ); |
+ if( iPos>=nCol ){ |
+ rc = FTS5_CORRUPT; |
+ break; |
+ } |
+ pCsr->aDoc[iPos]++; |
+ } |
+ } |
+ break; |
+ |
+ default: |
+ assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE ); |
+ pCsr->aDoc[0]++; |
+ break; |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5IterNextScan(pCsr->pIter); |
+ } |
+ |
+ if( rc==SQLITE_OK ){ |
+ zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); |
+ if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){ |
+ break; |
+ } |
+ if( sqlite3Fts5IterEof(pCsr->pIter) ) break; |
+ } |
+ } |
+ } |
+ } |
+ |
+ if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ |
+ while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++; |
+ assert( pCsr->iCol<pCsr->pConfig->nCol ); |
+ } |
+ return rc; |
+} |
+ |
+/* |
+** This is the xFilter implementation for the virtual table. |
+*/ |
+static int fts5VocabFilterMethod( |
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ |
+ int idxNum, /* Strategy index */ |
+ const char *zUnused, /* Unused */ |
+ int nUnused, /* Number of elements in apVal */ |
+ sqlite3_value **apVal /* Arguments for the indexing scheme */ |
+){ |
+ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; |
+ int rc = SQLITE_OK; |
+ |
+ int iVal = 0; |
+ int f = FTS5INDEX_QUERY_SCAN; |
+ const char *zTerm = 0; |
+ int nTerm = 0; |
+ |
+ sqlite3_value *pEq = 0; |
+ sqlite3_value *pGe = 0; |
+ sqlite3_value *pLe = 0; |
+ |
+ UNUSED_PARAM2(zUnused, nUnused); |
+ |
+ fts5VocabResetCursor(pCsr); |
+ if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++]; |
+ if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++]; |
+ if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++]; |
+ |
+ if( pEq ){ |
+ zTerm = (const char *)sqlite3_value_text(pEq); |
+ nTerm = sqlite3_value_bytes(pEq); |
+ f = 0; |
+ }else{ |
+ if( pGe ){ |
+ zTerm = (const char *)sqlite3_value_text(pGe); |
+ nTerm = sqlite3_value_bytes(pGe); |
+ } |
+ if( pLe ){ |
+ const char *zCopy = (const char *)sqlite3_value_text(pLe); |
+ pCsr->nLeTerm = sqlite3_value_bytes(pLe); |
+ pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1); |
+ if( pCsr->zLeTerm==0 ){ |
+ rc = SQLITE_NOMEM; |
+ }else{ |
+ memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1); |
+ } |
+ } |
+ } |
+ |
+ |
+ if( rc==SQLITE_OK ){ |
+ rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter); |
+ } |
+ if( rc==SQLITE_OK ){ |
+ rc = fts5VocabNextMethod(pCursor); |
+ } |
+ |
+ return rc; |
+} |
+ |
+/* |
+** This is the xEof method of the virtual table. SQLite calls this |
+** routine to find out if it has reached the end of a result set. |
+*/ |
+static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){ |
+ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; |
+ return pCsr->bEof; |
+} |
+ |
+static int fts5VocabColumnMethod( |
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ |
+ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ |
+ int iCol /* Index of column to read value from */ |
+){ |
+ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; |
+ int eDetail = pCsr->pConfig->eDetail; |
+ int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType; |
+ i64 iVal = 0; |
+ |
+ if( iCol==0 ){ |
+ sqlite3_result_text( |
+ pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT |
+ ); |
+ }else if( eType==FTS5_VOCAB_COL ){ |
+ assert( iCol==1 || iCol==2 || iCol==3 ); |
+ if( iCol==1 ){ |
+ if( eDetail!=FTS5_DETAIL_NONE ){ |
+ const char *z = pCsr->pConfig->azCol[pCsr->iCol]; |
+ sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); |
+ } |
+ }else if( iCol==2 ){ |
+ iVal = pCsr->aDoc[pCsr->iCol]; |
+ }else{ |
+ iVal = pCsr->aCnt[pCsr->iCol]; |
+ } |
+ }else{ |
+ assert( iCol==1 || iCol==2 ); |
+ if( iCol==1 ){ |
+ iVal = pCsr->aDoc[0]; |
+ }else{ |
+ iVal = pCsr->aCnt[0]; |
+ } |
+ } |
+ |
+ if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); |
+ return SQLITE_OK; |
+} |
+ |
+/* |
+** This is the xRowid method. The SQLite core calls this routine to |
+** retrieve the rowid for the current row of the result set. The |
+** rowid should be written to *pRowid. |
+*/ |
+static int fts5VocabRowidMethod( |
+ sqlite3_vtab_cursor *pCursor, |
+ sqlite_int64 *pRowid |
+){ |
+ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; |
+ *pRowid = pCsr->rowid; |
+ return SQLITE_OK; |
+} |
+ |
+static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ |
+ static const sqlite3_module fts5Vocab = { |
+ /* iVersion */ 2, |
+ /* xCreate */ fts5VocabCreateMethod, |
+ /* xConnect */ fts5VocabConnectMethod, |
+ /* xBestIndex */ fts5VocabBestIndexMethod, |
+ /* xDisconnect */ fts5VocabDisconnectMethod, |
+ /* xDestroy */ fts5VocabDestroyMethod, |
+ /* xOpen */ fts5VocabOpenMethod, |
+ /* xClose */ fts5VocabCloseMethod, |
+ /* xFilter */ fts5VocabFilterMethod, |
+ /* xNext */ fts5VocabNextMethod, |
+ /* xEof */ fts5VocabEofMethod, |
+ /* xColumn */ fts5VocabColumnMethod, |
+ /* xRowid */ fts5VocabRowidMethod, |
+ /* xUpdate */ 0, |
+ /* xBegin */ 0, |
+ /* xSync */ 0, |
+ /* xCommit */ 0, |
+ /* xRollback */ 0, |
+ /* xFindFunction */ 0, |
+ /* xRename */ 0, |
+ /* xSavepoint */ 0, |
+ /* xRelease */ 0, |
+ /* xRollbackTo */ 0, |
+ }; |
+ void *p = (void*)pGlobal; |
+ |
+ return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0); |
+} |
+ |
+ |
+ |
+ |
+ |
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */ |
+ |
+/************** End of fts5.c ************************************************/ |