| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 ** 2006 July 10 | |
| 3 ** | |
| 4 ** The author disclaims copyright to this source code. | |
| 5 ** | |
| 6 ************************************************************************* | |
| 7 ** Defines the interface to tokenizers used by fulltext-search. There | |
| 8 ** are three basic components: | |
| 9 ** | |
| 10 ** sqlite3_tokenizer_module is a singleton defining the tokenizer | |
| 11 ** interface functions. This is essentially the class structure for | |
| 12 ** tokenizers. | |
| 13 ** | |
| 14 ** sqlite3_tokenizer is used to define a particular tokenizer, perhaps | |
| 15 ** including customization information defined at creation time. | |
| 16 ** | |
| 17 ** sqlite3_tokenizer_cursor is generated by a tokenizer to generate | |
| 18 ** tokens from a particular input. | |
| 19 */ | |
| 20 #ifndef _FTS3_TOKENIZER_H_ | |
| 21 #define _FTS3_TOKENIZER_H_ | |
| 22 | |
| 23 /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. | |
| 24 ** If tokenizers are to be allowed to call sqlite3_*() functions, then | |
| 25 ** we will need a way to register the API consistently. | |
| 26 */ | |
| 27 #include "sqlite3.h" | |
| 28 | |
| 29 /* | |
| 30 ** Structures used by the tokenizer interface. When a new tokenizer | |
| 31 ** implementation is registered, the caller provides a pointer to | |
| 32 ** an sqlite3_tokenizer_module containing pointers to the callback | |
| 33 ** functions that make up an implementation. | |
| 34 ** | |
| 35 ** When an fts3 table is created, it passes any arguments passed to | |
| 36 ** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the | |
| 37 ** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer | |
| 38 ** implementation. The xCreate() function in turn returns an | |
| 39 ** sqlite3_tokenizer structure representing the specific tokenizer to | |
| 40 ** be used for the fts3 table (customized by the tokenizer clause arguments). | |
| 41 ** | |
| 42 ** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() | |
| 43 ** method is called. It returns an sqlite3_tokenizer_cursor object | |
| 44 ** that may be used to tokenize a specific input buffer based on | |
| 45 ** the tokenization rules supplied by a specific sqlite3_tokenizer | |
| 46 ** object. | |
| 47 */ | |
| 48 typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; | |
| 49 typedef struct sqlite3_tokenizer sqlite3_tokenizer; | |
| 50 typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; | |
| 51 | |
| 52 struct sqlite3_tokenizer_module { | |
| 53 | |
| 54 /* | |
| 55 ** Structure version. Should always be set to 0 or 1. | |
| 56 */ | |
| 57 int iVersion; | |
| 58 | |
| 59 /* | |
| 60 ** Create a new tokenizer. The values in the argv[] array are the | |
| 61 ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL | |
| 62 ** TABLE statement that created the fts3 table. For example, if | |
| 63 ** the following SQL is executed: | |
| 64 ** | |
| 65 ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2) | |
| 66 ** | |
| 67 ** then argc is set to 2, and the argv[] array contains pointers | |
| 68 ** to the strings "arg1" and "arg2". | |
| 69 ** | |
| 70 ** This method should return either SQLITE_OK (0), or an SQLite error | |
| 71 ** code. If SQLITE_OK is returned, then *ppTokenizer should be set | |
| 72 ** to point at the newly created tokenizer structure. The generic | |
| 73 ** sqlite3_tokenizer.pModule variable should not be initialized by | |
| 74 ** this callback. The caller will do so. | |
| 75 */ | |
| 76 int (*xCreate)( | |
| 77 int argc, /* Size of argv array */ | |
| 78 const char *const*argv, /* Tokenizer argument strings */ | |
| 79 sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ | |
| 80 ); | |
| 81 | |
| 82 /* | |
| 83 ** Destroy an existing tokenizer. The fts3 module calls this method | |
| 84 ** exactly once for each successful call to xCreate(). | |
| 85 */ | |
| 86 int (*xDestroy)(sqlite3_tokenizer *pTokenizer); | |
| 87 | |
| 88 /* | |
| 89 ** Create a tokenizer cursor to tokenize an input buffer. The caller | |
| 90 ** is responsible for ensuring that the input buffer remains valid | |
| 91 ** until the cursor is closed (using the xClose() method). | |
| 92 */ | |
| 93 int (*xOpen)( | |
| 94 sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ | |
| 95 const char *pInput, int nBytes, /* Input buffer */ | |
| 96 sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ | |
| 97 ); | |
| 98 | |
| 99 /* | |
| 100 ** Destroy an existing tokenizer cursor. The fts3 module calls this | |
| 101 ** method exactly once for each successful call to xOpen(). | |
| 102 */ | |
| 103 int (*xClose)(sqlite3_tokenizer_cursor *pCursor); | |
| 104 | |
| 105 /* | |
| 106 ** Retrieve the next token from the tokenizer cursor pCursor. This | |
| 107 ** method should either return SQLITE_OK and set the values of the | |
| 108 ** "OUT" variables identified below, or SQLITE_DONE to indicate that | |
| 109 ** the end of the buffer has been reached, or an SQLite error code. | |
| 110 ** | |
| 111 ** *ppToken should be set to point at a buffer containing the | |
| 112 ** normalized version of the token (i.e. after any case-folding and/or | |
| 113 ** stemming has been performed). *pnBytes should be set to the length | |
| 114 ** of this buffer in bytes. The input text that generated the token is | |
| 115 ** identified by the byte offsets returned in *piStartOffset and | |
| 116 ** *piEndOffset. *piStartOffset should be set to the index of the first | |
| 117 ** byte of the token in the input buffer. *piEndOffset should be set | |
| 118 ** to the index of the first byte just past the end of the token in | |
| 119 ** the input buffer. | |
| 120 ** | |
| 121 ** The buffer *ppToken is set to point at is managed by the tokenizer | |
| 122 ** implementation. It is only required to be valid until the next call | |
| 123 ** to xNext() or xClose(). | |
| 124 */ | |
| 125 /* TODO(shess) current implementation requires pInput to be | |
| 126 ** nul-terminated. This should either be fixed, or pInput/nBytes | |
| 127 ** should be converted to zInput. | |
| 128 */ | |
| 129 int (*xNext)( | |
| 130 sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ | |
| 131 const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ | |
| 132 int *piStartOffset, /* OUT: Byte offset of token in input buffer */ | |
| 133 int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ | |
| 134 int *piPosition /* OUT: Number of tokens returned before this one */ | |
| 135 ); | |
| 136 | |
| 137 /*********************************************************************** | |
| 138 ** Methods below this point are only available if iVersion>=1. | |
| 139 */ | |
| 140 | |
| 141 /* | |
| 142 ** Configure the language id of a tokenizer cursor. | |
| 143 */ | |
| 144 int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); | |
| 145 }; | |
| 146 | |
| 147 struct sqlite3_tokenizer { | |
| 148 const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ | |
| 149 /* Tokenizer implementations will typically add additional fields */ | |
| 150 }; | |
| 151 | |
| 152 struct sqlite3_tokenizer_cursor { | |
| 153 sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ | |
| 154 /* Tokenizer implementations will typically add additional fields */ | |
| 155 }; | |
| 156 | |
| 157 int fts3_global_term_cnt(int iTerm, int iCol); | |
| 158 int fts3_term_cnt(int iTerm, int iCol); | |
| 159 | |
| 160 | |
| 161 #endif /* _FTS3_TOKENIZER_H_ */ | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2363173002:
[sqlite] Remove obsolete reference version 3.8.7.4. (Closed)
