Import SQLite 3.8.7.4.

third_party/sqlite/src/ext/fts3/fts3_tokenizer.h

 /*
 ** 2006 July 10
 **
 ** The author disclaims copyright to this source code.
 **
 *************************************************************************
 ** Defines the interface to tokenizers used by fulltext-search.  There
 ** are three basic components:
 **
 ** sqlite3_tokenizer_module is a singleton defining the tokenizer
@@ skipping 34 matching lines @@
 ** the tokenization rules supplied by a specific sqlite3_tokenizer
 ** object.
 */
 typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
 typedef struct sqlite3_tokenizer sqlite3_tokenizer;
 typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
 
 struct sqlite3_tokenizer_module {
 
   /*
-  ** Structure version. Should always be set to 0.
+  ** Structure version. Should always be set to 0 or 1.
   */
   int iVersion;
 
   /*
   ** Create a new tokenizer. The values in the argv[] array are the
   ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
   ** TABLE statement that created the fts3 table. For example, if
   ** the following SQL is executed:
   **
   **   CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
   **
   ** then argc is set to 2, and the argv[] array contains pointers
   ** to the strings "arg1" and "arg2".
   **
   ** This method should return either SQLITE_OK (0), or an SQLite error 
   ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
   ** to point at the newly created tokenizer structure. The generic
-  ** sqlite3_tokenizer.pModule variable should not be initialised by
+  ** sqlite3_tokenizer.pModule variable should not be initialized by
   ** this callback. The caller will do so.
   */
   int (*xCreate)(
     int argc,                           /* Size of argv array */
     const char *const*argv,             /* Tokenizer argument strings */
     sqlite3_tokenizer **ppTokenizer     /* OUT: Created tokenizer */
   );
 
   /*
   ** Destroy an existing tokenizer. The fts3 module calls this method
@@ skipping 42 matching lines @@
   ** nul-terminated.  This should either be fixed, or pInput/nBytes
   ** should be converted to zInput.
   */
   int (*xNext)(
     sqlite3_tokenizer_cursor *pCursor,   /* Tokenizer cursor */
     const char **ppToken, int *pnBytes,  /* OUT: Normalized text for token */
     int *piStartOffset,  /* OUT: Byte offset of token in input buffer */
     int *piEndOffset,    /* OUT: Byte offset of end of token in input buffer */
     int *piPosition      /* OUT: Number of tokens returned before this one */
   );
+
+  /***********************************************************************
+  ** Methods below this point are only available if iVersion>=1.
+  */
+
+  /* 
+  ** Configure the language id of a tokenizer cursor.
+  */
+  int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid);
 };
 
 struct sqlite3_tokenizer {
   const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
   /* Tokenizer implementations will typically add additional fields */
 };
 
 struct sqlite3_tokenizer_cursor {
   sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
   /* Tokenizer implementations will typically add additional fields */
 };
 
 int fts3_global_term_cnt(int iTerm, int iCol);
 int fts3_term_cnt(int iTerm, int iCol);
 
 
 #endif /* _FTS3_TOKENIZER_H_ */