Import SQLite 3.8.7.4.

third_party/sqlite/src/ext/fts3/fts3_tokenizer1.c

 /*
 ** 2006 Oct 10
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 ******************************************************************************
 **
 ** Implementation of the "simple" full-text-search tokenizer.
 */
 
 /*
 ** The code in this file is only compiled if:
 **
 **     * The FTS3 module is being built as an extension
 **       (in which case SQLITE_CORE is not defined), or
 **
 **     * The FTS3 module is being built into the core of
 **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
 */
+#include "fts3Int.h"
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
 
-#include "fts3Int.h"
-
 #include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
 
 #include "fts3_tokenizer.h"
 
 typedef struct simple_tokenizer {
   sqlite3_tokenizer base;
   char delim[128];             /* flag ASCII delimiters */
@@ skipping 173 matching lines @@
 /*
 ** The set of routines that implement the simple tokenizer
 */
 static const sqlite3_tokenizer_module simpleTokenizerModule = {
   0,
   simpleCreate,
   simpleDestroy,
   simpleOpen,
   simpleClose,
   simpleNext,
+  0,
 };
 
 /*
 ** Allocate a new simple tokenizer.  Return a pointer to the new
 ** tokenizer in *ppModule
 */
 void sqlite3Fts3SimpleTokenizerModule(
   sqlite3_tokenizer_module const**ppModule
 ){
   *ppModule = &simpleTokenizerModule;
 }
 
 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */