Import SQLite 3.8.7.4.

third_party/sqlite/src/ext/fts3/fts3_tokenizer.h

Index: third_party/sqlite/src/ext/fts3/fts3_tokenizer.h
diff --git a/third_party/sqlite/src/ext/fts3/fts3_tokenizer.h b/third_party/sqlite/src/ext/fts3/fts3_tokenizer.h
index 615644506cb75c42358e0ab6eef30db3c3eae954..4a40b2b38503833bf17ded9ee2e2742e86267fc6 100644
--- a/third_party/sqlite/src/ext/fts3/fts3_tokenizer.h
+++ b/third_party/sqlite/src/ext/fts3/fts3_tokenizer.h
@@ -52,7 +52,7 @@ typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
 struct sqlite3_tokenizer_module {
 
   /*
-  ** Structure version. Should always be set to 0.
+  ** Structure version. Should always be set to 0 or 1.
   */
   int iVersion;
 
@@ -70,7 +70,7 @@ struct sqlite3_tokenizer_module {
   ** This method should return either SQLITE_OK (0), or an SQLite error 
   ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
   ** to point at the newly created tokenizer structure. The generic
-  ** sqlite3_tokenizer.pModule variable should not be initialised by
+  ** sqlite3_tokenizer.pModule variable should not be initialized by
   ** this callback. The caller will do so.
   */
   int (*xCreate)(
@@ -133,6 +133,15 @@ struct sqlite3_tokenizer_module {
     int *piEndOffset,    /* OUT: Byte offset of end of token in input buffer */
     int *piPosition      /* OUT: Number of tokens returned before this one */
   );
+
+  /***********************************************************************
+  ** Methods below this point are only available if iVersion>=1.
+  */
+
+  /* 
+  ** Configure the language id of a tokenizer cursor.
+  */
+  int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid);
 };
 
 struct sqlite3_tokenizer {