| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 ** 2014 May 31 | |
| 3 ** | |
| 4 ** The author disclaims copyright to this source code. In place of | |
| 5 ** a legal notice, here is a blessing: | |
| 6 ** | |
| 7 ** May you do good and not evil. | |
| 8 ** May you find forgiveness for yourself and forgive others. | |
| 9 ** May you share freely, never taking more than you give. | |
| 10 ** | |
| 11 ****************************************************************************** | |
| 12 ** | |
| 13 ** Interfaces to extend FTS5. Using the interfaces defined in this file, | |
| 14 ** FTS5 may be extended with: | |
| 15 ** | |
| 16 ** * custom tokenizers, and | |
| 17 ** * custom auxiliary functions. | |
| 18 */ | |
| 19 | |
| 20 | |
| 21 #ifndef _FTS5_H | |
| 22 #define _FTS5_H | |
| 23 | |
| 24 #include "sqlite3.h" | |
| 25 | |
| 26 #ifdef __cplusplus | |
| 27 extern "C" { | |
| 28 #endif | |
| 29 | |
| 30 /************************************************************************* | |
| 31 ** CUSTOM AUXILIARY FUNCTIONS | |
| 32 ** | |
| 33 ** Virtual table implementations may overload SQL functions by implementing | |
| 34 ** the sqlite3_module.xFindFunction() method. | |
| 35 */ | |
| 36 | |
| 37 typedef struct Fts5ExtensionApi Fts5ExtensionApi; | |
| 38 typedef struct Fts5Context Fts5Context; | |
| 39 typedef struct Fts5PhraseIter Fts5PhraseIter; | |
| 40 | |
| 41 typedef void (*fts5_extension_function)( | |
| 42 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ | |
| 43 Fts5Context *pFts, /* First arg to pass to pApi functions */ | |
| 44 sqlite3_context *pCtx, /* Context for returning result/error */ | |
| 45 int nVal, /* Number of values in apVal[] array */ | |
| 46 sqlite3_value **apVal /* Array of trailing arguments */ | |
| 47 ); | |
| 48 | |
| 49 struct Fts5PhraseIter { | |
| 50 const unsigned char *a; | |
| 51 const unsigned char *b; | |
| 52 }; | |
| 53 | |
| 54 /* | |
| 55 ** EXTENSION API FUNCTIONS | |
| 56 ** | |
| 57 ** xUserData(pFts): | |
| 58 ** Return a copy of the context pointer the extension function was | |
| 59 ** registered with. | |
| 60 ** | |
| 61 ** xColumnTotalSize(pFts, iCol, pnToken): | |
| 62 ** If parameter iCol is less than zero, set output variable *pnToken | |
| 63 ** to the total number of tokens in the FTS5 table. Or, if iCol is | |
| 64 ** non-negative but less than the number of columns in the table, return | |
| 65 ** the total number of tokens in column iCol, considering all rows in | |
| 66 ** the FTS5 table. | |
| 67 ** | |
| 68 ** If parameter iCol is greater than or equal to the number of columns | |
| 69 ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. | |
| 70 ** an OOM condition or IO error), an appropriate SQLite error code is | |
| 71 ** returned. | |
| 72 ** | |
| 73 ** xColumnCount(pFts): | |
| 74 ** Return the number of columns in the table. | |
| 75 ** | |
| 76 ** xColumnSize(pFts, iCol, pnToken): | |
| 77 ** If parameter iCol is less than zero, set output variable *pnToken | |
| 78 ** to the total number of tokens in the current row. Or, if iCol is | |
| 79 ** non-negative but less than the number of columns in the table, set | |
| 80 ** *pnToken to the number of tokens in column iCol of the current row. | |
| 81 ** | |
| 82 ** If parameter iCol is greater than or equal to the number of columns | |
| 83 ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. | |
| 84 ** an OOM condition or IO error), an appropriate SQLite error code is | |
| 85 ** returned. | |
| 86 ** | |
| 87 ** xColumnText: | |
| 88 ** This function attempts to retrieve the text of column iCol of the | |
| 89 ** current document. If successful, (*pz) is set to point to a buffer | |
| 90 ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes | |
| 91 ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, | |
| 92 ** if an error occurs, an SQLite error code is returned and the final values | |
| 93 ** of (*pz) and (*pn) are undefined. | |
| 94 ** | |
| 95 ** xPhraseCount: | |
| 96 ** Returns the number of phrases in the current query expression. | |
| 97 ** | |
| 98 ** xPhraseSize: | |
| 99 ** Returns the number of tokens in phrase iPhrase of the query. Phrases | |
| 100 ** are numbered starting from zero. | |
| 101 ** | |
| 102 ** xInstCount: | |
| 103 ** Set *pnInst to the total number of occurrences of all phrases within | |
| 104 ** the query within the current row. Return SQLITE_OK if successful, or | |
| 105 ** an error code (i.e. SQLITE_NOMEM) if an error occurs. | |
| 106 ** | |
| 107 ** xInst: | |
| 108 ** Query for the details of phrase match iIdx within the current row. | |
| 109 ** Phrase matches are numbered starting from zero, so the iIdx argument | |
| 110 ** should be greater than or equal to zero and smaller than the value | |
| 111 ** output by xInstCount(). | |
| 112 ** | |
| 113 ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) | |
| 114 ** if an error occurs. | |
| 115 ** | |
| 116 ** xRowid: | |
| 117 ** Returns the rowid of the current row. | |
| 118 ** | |
| 119 ** xTokenize: | |
| 120 ** Tokenize text using the tokenizer belonging to the FTS5 table. | |
| 121 ** | |
| 122 ** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): | |
| 123 ** This API function is used to query the FTS table for phrase iPhrase | |
| 124 ** of the current query. Specifically, a query equivalent to: | |
| 125 ** | |
| 126 ** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid | |
| 127 ** | |
| 128 ** with $p set to a phrase equivalent to the phrase iPhrase of the | |
| 129 ** current query is executed. For each row visited, the callback function | |
| 130 ** passed as the fourth argument is invoked. The context and API objects | |
| 131 ** passed to the callback function may be used to access the properties of | |
| 132 ** each matched row. Invoking Api.xUserData() returns a copy of the pointer | |
| 133 ** passed as the third argument to pUserData. | |
| 134 ** | |
| 135 ** If the callback function returns any value other than SQLITE_OK, the | |
| 136 ** query is abandoned and the xQueryPhrase function returns immediately. | |
| 137 ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. | |
| 138 ** Otherwise, the error code is propagated upwards. | |
| 139 ** | |
| 140 ** If the query runs to completion without incident, SQLITE_OK is returned. | |
| 141 ** Or, if some error occurs before the query completes or is aborted by | |
| 142 ** the callback, an SQLite error code is returned. | |
| 143 ** | |
| 144 ** | |
| 145 ** xSetAuxdata(pFts5, pAux, xDelete) | |
| 146 ** | |
| 147 ** Save the pointer passed as the second argument as the extension functions | |
| 148 ** "auxiliary data". The pointer may then be retrieved by the current or any | |
| 149 ** future invocation of the same fts5 extension function made as part of | |
| 150 ** of the same MATCH query using the xGetAuxdata() API. | |
| 151 ** | |
| 152 ** Each extension function is allocated a single auxiliary data slot for | |
| 153 ** each FTS query (MATCH expression). If the extension function is invoked | |
| 154 ** more than once for a single FTS query, then all invocations share a | |
| 155 ** single auxiliary data context. | |
| 156 ** | |
| 157 ** If there is already an auxiliary data pointer when this function is | |
| 158 ** invoked, then it is replaced by the new pointer. If an xDelete callback | |
| 159 ** was specified along with the original pointer, it is invoked at this | |
| 160 ** point. | |
| 161 ** | |
| 162 ** The xDelete callback, if one is specified, is also invoked on the | |
| 163 ** auxiliary data pointer after the FTS5 query has finished. | |
| 164 ** | |
| 165 ** If an error (e.g. an OOM condition) occurs within this function, an | |
| 166 ** the auxiliary data is set to NULL and an error code returned. If the | |
| 167 ** xDelete parameter was not NULL, it is invoked on the auxiliary data | |
| 168 ** pointer before returning. | |
| 169 ** | |
| 170 ** | |
| 171 ** xGetAuxdata(pFts5, bClear) | |
| 172 ** | |
| 173 ** Returns the current auxiliary data pointer for the fts5 extension | |
| 174 ** function. See the xSetAuxdata() method for details. | |
| 175 ** | |
| 176 ** If the bClear argument is non-zero, then the auxiliary data is cleared | |
| 177 ** (set to NULL) before this function returns. In this case the xDelete, | |
| 178 ** if any, is not invoked. | |
| 179 ** | |
| 180 ** | |
| 181 ** xRowCount(pFts5, pnRow) | |
| 182 ** | |
| 183 ** This function is used to retrieve the total number of rows in the table. | |
| 184 ** In other words, the same value that would be returned by: | |
| 185 ** | |
| 186 ** SELECT count(*) FROM ftstable; | |
| 187 ** | |
| 188 ** xPhraseFirst() | |
| 189 ** This function is used, along with type Fts5PhraseIter and the xPhraseNext | |
| 190 ** method, to iterate through all instances of a single query phrase within | |
| 191 ** the current row. This is the same information as is accessible via the | |
| 192 ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient | |
| 193 ** to use, this API may be faster under some circumstances. To iterate | |
| 194 ** through instances of phrase iPhrase, use the following code: | |
| 195 ** | |
| 196 ** Fts5PhraseIter iter; | |
| 197 ** int iCol, iOff; | |
| 198 ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); | |
| 199 ** iOff>=0; | |
| 200 ** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) | |
| 201 ** ){ | |
| 202 ** // An instance of phrase iPhrase at offset iOff of column iCol | |
| 203 ** } | |
| 204 ** | |
| 205 ** The Fts5PhraseIter structure is defined above. Applications should not | |
| 206 ** modify this structure directly - it should only be used as shown above | |
| 207 ** with the xPhraseFirst() and xPhraseNext() API methods. | |
| 208 ** | |
| 209 ** xPhraseNext() | |
| 210 ** See xPhraseFirst above. | |
| 211 */ | |
| 212 struct Fts5ExtensionApi { | |
| 213 int iVersion; /* Currently always set to 1 */ | |
| 214 | |
| 215 void *(*xUserData)(Fts5Context*); | |
| 216 | |
| 217 int (*xColumnCount)(Fts5Context*); | |
| 218 int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); | |
| 219 int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); | |
| 220 | |
| 221 int (*xTokenize)(Fts5Context*, | |
| 222 const char *pText, int nText, /* Text to tokenize */ | |
| 223 void *pCtx, /* Context passed to xToken() */ | |
| 224 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ | |
| 225 ); | |
| 226 | |
| 227 int (*xPhraseCount)(Fts5Context*); | |
| 228 int (*xPhraseSize)(Fts5Context*, int iPhrase); | |
| 229 | |
| 230 int (*xInstCount)(Fts5Context*, int *pnInst); | |
| 231 int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); | |
| 232 | |
| 233 sqlite3_int64 (*xRowid)(Fts5Context*); | |
| 234 int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); | |
| 235 int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); | |
| 236 | |
| 237 int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, | |
| 238 int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) | |
| 239 ); | |
| 240 int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); | |
| 241 void *(*xGetAuxdata)(Fts5Context*, int bClear); | |
| 242 | |
| 243 void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); | |
| 244 void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); | |
| 245 }; | |
| 246 | |
| 247 /* | |
| 248 ** CUSTOM AUXILIARY FUNCTIONS | |
| 249 *************************************************************************/ | |
| 250 | |
| 251 /************************************************************************* | |
| 252 ** CUSTOM TOKENIZERS | |
| 253 ** | |
| 254 ** Applications may also register custom tokenizer types. A tokenizer | |
| 255 ** is registered by providing fts5 with a populated instance of the | |
| 256 ** following structure. All structure methods must be defined, setting | |
| 257 ** any member of the fts5_tokenizer struct to NULL leads to undefined | |
| 258 ** behaviour. The structure methods are expected to function as follows: | |
| 259 ** | |
| 260 ** xCreate: | |
| 261 ** This function is used to allocate and inititalize a tokenizer instance. | |
| 262 ** A tokenizer instance is required to actually tokenize text. | |
| 263 ** | |
| 264 ** The first argument passed to this function is a copy of the (void*) | |
| 265 ** pointer provided by the application when the fts5_tokenizer object | |
| 266 ** was registered with FTS5 (the third argument to xCreateTokenizer()). | |
| 267 ** The second and third arguments are an array of nul-terminated strings | |
| 268 ** containing the tokenizer arguments, if any, specified following the | |
| 269 ** tokenizer name as part of the CREATE VIRTUAL TABLE statement used | |
| 270 ** to create the FTS5 table. | |
| 271 ** | |
| 272 ** The final argument is an output variable. If successful, (*ppOut) | |
| 273 ** should be set to point to the new tokenizer handle and SQLITE_OK | |
| 274 ** returned. If an error occurs, some value other than SQLITE_OK should | |
| 275 ** be returned. In this case, fts5 assumes that the final value of *ppOut | |
| 276 ** is undefined. | |
| 277 ** | |
| 278 ** xDelete: | |
| 279 ** This function is invoked to delete a tokenizer handle previously | |
| 280 ** allocated using xCreate(). Fts5 guarantees that this function will | |
| 281 ** be invoked exactly once for each successful call to xCreate(). | |
| 282 ** | |
| 283 ** xTokenize: | |
| 284 ** This function is expected to tokenize the nText byte string indicated | |
| 285 ** by argument pText. pText may or may not be nul-terminated. The first | |
| 286 ** argument passed to this function is a pointer to an Fts5Tokenizer object | |
| 287 ** returned by an earlier call to xCreate(). | |
| 288 ** | |
| 289 ** The second argument indicates the reason that FTS5 is requesting | |
| 290 ** tokenization of the supplied text. This is always one of the following | |
| 291 ** four values: | |
| 292 ** | |
| 293 ** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into | |
| 294 ** or removed from the FTS table. The tokenizer is being invoked to | |
| 295 ** determine the set of tokens to add to (or delete from) the | |
| 296 ** FTS index. | |
| 297 ** | |
| 298 ** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed | |
| 299 ** against the FTS index. The tokenizer is being called to tokenize | |
| 300 ** a bareword or quoted string specified as part of the query. | |
| 301 ** | |
| 302 ** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as | |
| 303 ** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is | |
| 304 ** followed by a "*" character, indicating that the last token | |
| 305 ** returned by the tokenizer will be treated as a token prefix. | |
| 306 ** | |
| 307 ** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to | |
| 308 ** satisfy an fts5_api.xTokenize() request made by an auxiliary | |
| 309 ** function. Or an fts5_api.xColumnSize() request made by the same | |
| 310 ** on a columnsize=0 database. | |
| 311 ** </ul> | |
| 312 ** | |
| 313 ** For each token in the input string, the supplied callback xToken() must | |
| 314 ** be invoked. The first argument to it should be a copy of the pointer | |
| 315 ** passed as the second argument to xTokenize(). The third and fourth | |
| 316 ** arguments are a pointer to a buffer containing the token text, and the | |
| 317 ** size of the token in bytes. The 4th and 5th arguments are the byte offsets | |
| 318 ** of the first byte of and first byte immediately following the text from | |
| 319 ** which the token is derived within the input. | |
| 320 ** | |
| 321 ** The second argument passed to the xToken() callback ("tflags") should | |
| 322 ** normally be set to 0. The exception is if the tokenizer supports | |
| 323 ** synonyms. In this case see the discussion below for details. | |
| 324 ** | |
| 325 ** FTS5 assumes the xToken() callback is invoked for each token in the | |
| 326 ** order that they occur within the input text. | |
| 327 ** | |
| 328 ** If an xToken() callback returns any value other than SQLITE_OK, then | |
| 329 ** the tokenization should be abandoned and the xTokenize() method should | |
| 330 ** immediately return a copy of the xToken() return value. Or, if the | |
| 331 ** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, | |
| 332 ** if an error occurs with the xTokenize() implementation itself, it | |
| 333 ** may abandon the tokenization and return any error code other than | |
| 334 ** SQLITE_OK or SQLITE_DONE. | |
| 335 ** | |
| 336 ** SYNONYM SUPPORT | |
| 337 ** | |
| 338 ** Custom tokenizers may also support synonyms. Consider a case in which a | |
| 339 ** user wishes to query for a phrase such as "first place". Using the | |
| 340 ** built-in tokenizers, the FTS5 query 'first + place' will match instances | |
| 341 ** of "first place" within the document set, but not alternative forms | |
| 342 ** such as "1st place". In some applications, it would be better to match | |
| 343 ** all instances of "first place" or "1st place" regardless of which form | |
| 344 ** the user specified in the MATCH query text. | |
| 345 ** | |
| 346 ** There are several ways to approach this in FTS5: | |
| 347 ** | |
| 348 ** <ol><li> By mapping all synonyms to a single token. In this case, the | |
| 349 ** In the above example, this means that the tokenizer returns the | |
| 350 ** same token for inputs "first" and "1st". Say that token is in | |
| 351 ** fact "first", so that when the user inserts the document "I won | |
| 352 ** 1st place" entries are added to the index for tokens "i", "won", | |
| 353 ** "first" and "place". If the user then queries for '1st + place', | |
| 354 ** the tokenizer substitutes "first" for "1st" and the query works | |
| 355 ** as expected. | |
| 356 ** | |
| 357 ** <li> By adding multiple synonyms for a single term to the FTS index. | |
| 358 ** In this case, when tokenizing query text, the tokenizer may | |
| 359 ** provide multiple synonyms for a single term within the document. | |
| 360 ** FTS5 then queries the index for each synonym individually. For | |
| 361 ** example, faced with the query: | |
| 362 ** | |
| 363 ** <codeblock> | |
| 364 ** ... MATCH 'first place'</codeblock> | |
| 365 ** | |
| 366 ** the tokenizer offers both "1st" and "first" as synonyms for the | |
| 367 ** first token in the MATCH query and FTS5 effectively runs a query | |
| 368 ** similar to: | |
| 369 ** | |
| 370 ** <codeblock> | |
| 371 ** ... MATCH '(first OR 1st) place'</codeblock> | |
| 372 ** | |
| 373 ** except that, for the purposes of auxiliary functions, the query | |
| 374 ** still appears to contain just two phrases - "(first OR 1st)" | |
| 375 ** being treated as a single phrase. | |
| 376 ** | |
| 377 ** <li> By adding multiple synonyms for a single term to the FTS index. | |
| 378 ** Using this method, when tokenizing document text, the tokenizer | |
| 379 ** provides multiple synonyms for each token. So that when a | |
| 380 ** document such as "I won first place" is tokenized, entries are | |
| 381 ** added to the FTS index for "i", "won", "first", "1st" and | |
| 382 ** "place". | |
| 383 ** | |
| 384 ** This way, even if the tokenizer does not provide synonyms | |
| 385 ** when tokenizing query text (it should not - to do would be | |
| 386 ** inefficient), it doesn't matter if the user queries for | |
| 387 ** 'first + place' or '1st + place', as there are entires in the | |
| 388 ** FTS index corresponding to both forms of the first token. | |
| 389 ** </ol> | |
| 390 ** | |
| 391 ** Whether it is parsing document or query text, any call to xToken that | |
| 392 ** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit | |
| 393 ** is considered to supply a synonym for the previous token. For example, | |
| 394 ** when parsing the document "I won first place", a tokenizer that supports | |
| 395 ** synonyms would call xToken() 5 times, as follows: | |
| 396 ** | |
| 397 ** <codeblock> | |
| 398 ** xToken(pCtx, 0, "i", 1, 0, 1); | |
| 399 ** xToken(pCtx, 0, "won", 3, 2, 5); | |
| 400 ** xToken(pCtx, 0, "first", 5, 6, 11); | |
| 401 ** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); | |
| 402 ** xToken(pCtx, 0, "place", 5, 12, 17); | |
| 403 **</codeblock> | |
| 404 ** | |
| 405 ** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time | |
| 406 ** xToken() is called. Multiple synonyms may be specified for a single token | |
| 407 ** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. | |
| 408 ** There is no limit to the number of synonyms that may be provided for a | |
| 409 ** single token. | |
| 410 ** | |
| 411 ** In many cases, method (1) above is the best approach. It does not add | |
| 412 ** extra data to the FTS index or require FTS5 to query for multiple terms, | |
| 413 ** so it is efficient in terms of disk space and query speed. However, it | |
| 414 ** does not support prefix queries very well. If, as suggested above, the | |
| 415 ** token "first" is subsituted for "1st" by the tokenizer, then the query: | |
| 416 ** | |
| 417 ** <codeblock> | |
| 418 ** ... MATCH '1s*'</codeblock> | |
| 419 ** | |
| 420 ** will not match documents that contain the token "1st" (as the tokenizer | |
| 421 ** will probably not map "1s" to any prefix of "first"). | |
| 422 ** | |
| 423 ** For full prefix support, method (3) may be preferred. In this case, | |
| 424 ** because the index contains entries for both "first" and "1st", prefix | |
| 425 ** queries such as 'fi*' or '1s*' will match correctly. However, because | |
| 426 ** extra entries are added to the FTS index, this method uses more space | |
| 427 ** within the database. | |
| 428 ** | |
| 429 ** Method (2) offers a midpoint between (1) and (3). Using this method, | |
| 430 ** a query such as '1s*' will match documents that contain the literal | |
| 431 ** token "1st", but not "first" (assuming the tokenizer is not able to | |
| 432 ** provide synonyms for prefixes). However, a non-prefix query like '1st' | |
| 433 ** will match against "1st" and "first". This method does not require | |
| 434 ** extra disk space, as no extra entries are added to the FTS index. | |
| 435 ** On the other hand, it may require more CPU cycles to run MATCH queries, | |
| 436 ** as separate queries of the FTS index are required for each synonym. | |
| 437 ** | |
| 438 ** When using methods (2) or (3), it is important that the tokenizer only | |
| 439 ** provide synonyms when tokenizing document text (method (2)) or query | |
| 440 ** text (method (3)), not both. Doing so will not cause any errors, but is | |
| 441 ** inefficient. | |
| 442 */ | |
| 443 typedef struct Fts5Tokenizer Fts5Tokenizer; | |
| 444 typedef struct fts5_tokenizer fts5_tokenizer; | |
| 445 struct fts5_tokenizer { | |
| 446 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); | |
| 447 void (*xDelete)(Fts5Tokenizer*); | |
| 448 int (*xTokenize)(Fts5Tokenizer*, | |
| 449 void *pCtx, | |
| 450 int flags, /* Mask of FTS5_TOKENIZE_* flags */ | |
| 451 const char *pText, int nText, | |
| 452 int (*xToken)( | |
| 453 void *pCtx, /* Copy of 2nd argument to xTokenize() */ | |
| 454 int tflags, /* Mask of FTS5_TOKEN_* flags */ | |
| 455 const char *pToken, /* Pointer to buffer containing token */ | |
| 456 int nToken, /* Size of token in bytes */ | |
| 457 int iStart, /* Byte offset of token within input text */ | |
| 458 int iEnd /* Byte offset of end of token within input text */ | |
| 459 ) | |
| 460 ); | |
| 461 }; | |
| 462 | |
| 463 /* Flags that may be passed as the third argument to xTokenize() */ | |
| 464 #define FTS5_TOKENIZE_QUERY 0x0001 | |
| 465 #define FTS5_TOKENIZE_PREFIX 0x0002 | |
| 466 #define FTS5_TOKENIZE_DOCUMENT 0x0004 | |
| 467 #define FTS5_TOKENIZE_AUX 0x0008 | |
| 468 | |
| 469 /* Flags that may be passed by the tokenizer implementation back to FTS5 | |
| 470 ** as the third argument to the supplied xToken callback. */ | |
| 471 #define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ | |
| 472 | |
| 473 /* | |
| 474 ** END OF CUSTOM TOKENIZERS | |
| 475 *************************************************************************/ | |
| 476 | |
| 477 /************************************************************************* | |
| 478 ** FTS5 EXTENSION REGISTRATION API | |
| 479 */ | |
| 480 typedef struct fts5_api fts5_api; | |
| 481 struct fts5_api { | |
| 482 int iVersion; /* Currently always set to 2 */ | |
| 483 | |
| 484 /* Create a new tokenizer */ | |
| 485 int (*xCreateTokenizer)( | |
| 486 fts5_api *pApi, | |
| 487 const char *zName, | |
| 488 void *pContext, | |
| 489 fts5_tokenizer *pTokenizer, | |
| 490 void (*xDestroy)(void*) | |
| 491 ); | |
| 492 | |
| 493 /* Find an existing tokenizer */ | |
| 494 int (*xFindTokenizer)( | |
| 495 fts5_api *pApi, | |
| 496 const char *zName, | |
| 497 void **ppContext, | |
| 498 fts5_tokenizer *pTokenizer | |
| 499 ); | |
| 500 | |
| 501 /* Create a new auxiliary function */ | |
| 502 int (*xCreateFunction)( | |
| 503 fts5_api *pApi, | |
| 504 const char *zName, | |
| 505 void *pContext, | |
| 506 fts5_extension_function xFunction, | |
| 507 void (*xDestroy)(void*) | |
| 508 ); | |
| 509 }; | |
| 510 | |
| 511 /* | |
| 512 ** END OF REGISTRATION API | |
| 513 *************************************************************************/ | |
| 514 | |
| 515 #ifdef __cplusplus | |
| 516 } /* end of the 'extern "C"' block */ | |
| 517 #endif | |
| 518 | |
| 519 #endif /* _FTS5_H */ | |
| 520 | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2846743003:
[sql] Remove SQLite 3.10.2 reference directory. (Closed)
