| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 ** 2006 July 10 | |
| 3 ** | |
| 4 ** The author disclaims copyright to this source code. | |
| 5 ** | |
| 6 ************************************************************************* | |
| 7 ** Defines the interface to tokenizers used by fulltext-search. There | |
| 8 ** are three basic components: | |
| 9 ** | |
| 10 ** sqlite3_tokenizer_module is a singleton defining the tokenizer | |
| 11 ** interface functions. This is essentially the class structure for | |
| 12 ** tokenizers. | |
| 13 ** | |
| 14 ** sqlite3_tokenizer is used to define a particular tokenizer, perhaps | |
| 15 ** including customization information defined at creation time. | |
| 16 ** | |
| 17 ** sqlite3_tokenizer_cursor is generated by a tokenizer to generate | |
| 18 ** tokens from a particular input. | |
| 19 */ | |
| 20 #ifndef _TOKENIZER_H_ | |
| 21 #define _TOKENIZER_H_ | |
| 22 | |
| 23 /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. | |
| 24 ** If tokenizers are to be allowed to call sqlite3_*() functions, then | |
| 25 ** we will need a way to register the API consistently. | |
| 26 */ | |
| 27 #include "sqlite3.h" | |
| 28 | |
| 29 /* | |
| 30 ** Structures used by the tokenizer interface. | |
| 31 */ | |
| 32 typedef struct sqlite3_tokenizer sqlite3_tokenizer; | |
| 33 typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; | |
| 34 typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; | |
| 35 | |
| 36 struct sqlite3_tokenizer_module { | |
| 37 int iVersion; /* currently 0 */ | |
| 38 | |
| 39 /* | |
| 40 ** Create and destroy a tokenizer. argc/argv are passed down from | |
| 41 ** the fulltext virtual table creation to allow customization. | |
| 42 */ | |
| 43 int (*xCreate)(int argc, const char **argv, | |
| 44 sqlite3_tokenizer **ppTokenizer); | |
| 45 int (*xDestroy)(sqlite3_tokenizer *pTokenizer); | |
| 46 | |
| 47 /* | |
| 48 ** Tokenize a particular input. Call xOpen() to prepare to | |
| 49 ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then | |
| 50 ** xClose() to free any internal state. The pInput passed to | |
| 51 ** xOpen() must exist until the cursor is closed. The ppToken | |
| 52 ** result from xNext() is only valid until the next call to xNext() | |
| 53 ** or until xClose() is called. | |
| 54 */ | |
| 55 /* TODO(shess) current implementation requires pInput to be | |
| 56 ** nul-terminated. This should either be fixed, or pInput/nBytes | |
| 57 ** should be converted to zInput. | |
| 58 */ | |
| 59 int (*xOpen)(sqlite3_tokenizer *pTokenizer, | |
| 60 const char *pInput, int nBytes, | |
| 61 sqlite3_tokenizer_cursor **ppCursor); | |
| 62 int (*xClose)(sqlite3_tokenizer_cursor *pCursor); | |
| 63 int (*xNext)(sqlite3_tokenizer_cursor *pCursor, | |
| 64 const char **ppToken, int *pnBytes, | |
| 65 int *piStartOffset, int *piEndOffset, int *piPosition); | |
| 66 }; | |
| 67 | |
| 68 struct sqlite3_tokenizer { | |
| 69 sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ | |
| 70 /* Tokenizer implementations will typically add additional fields */ | |
| 71 }; | |
| 72 | |
| 73 struct sqlite3_tokenizer_cursor { | |
| 74 sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ | |
| 75 /* Tokenizer implementations will typically add additional fields */ | |
| 76 }; | |
| 77 | |
| 78 /* | |
| 79 ** Get the module for a tokenizer which generates tokens based on a | |
| 80 ** set of non-token characters. The default is to break tokens at any | |
| 81 ** non-alnum character, though the set of delimiters can also be | |
| 82 ** specified by the first argv argument to xCreate(). | |
| 83 */ | |
| 84 /* TODO(shess) This doesn't belong here. Need some sort of | |
| 85 ** registration process. | |
| 86 */ | |
| 87 void get_simple_tokenizer_module(sqlite3_tokenizer_module **ppModule); | |
| 88 | |
| 89 #endif /* _TOKENIZER_H_ */ | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2363173002:
[sqlite] Remove obsolete reference version 3.8.7.4. (Closed)
