Cleanup SQLite 3.6.18 import.

third_party/sqlite/src/ext/fts1/simple_tokenizer.c

 /*
 ** The author disclaims copyright to this source code.
 **
 *************************************************************************
 ** Implementation of the "simple" full-text-search tokenizer.
 */
 
 #include <assert.h>
 #if !defined(__APPLE__)
 #include <malloc.h>
@@ skipping 120 matching lines @@
     int n = (int) strcspn(c->pCurrent, t->zDelim);
     if( n>0 ){
       if( n+1>c->nTokenAllocated ){
         c->zToken = realloc(c->zToken, n+1);
       }
       for(ii=0; ii<n; ii++){
         /* TODO(shess) This needs expansion to handle UTF-8
         ** case-insensitivity.
         */
         char ch = c->pCurrent[ii];
-        c->zToken[ii] = ((ch>='A' && ch<='Z') ? (ch-'A'+'a') : ch);
+        c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch;
       }
       c->zToken[n] = '\0';
       *ppToken = c->zToken;
       *pnBytes = n;
       *piStartOffset = (int) (c->pCurrent-c->pInput);
       *piEndOffset = *piStartOffset+n;
       *piPosition = c->iToken++;
       c->pCurrent += n + 1;
 
       return SQLITE_OK;
@@ skipping 13 matching lines @@
   simpleOpen,
   simpleClose,
   simpleNext,
 };
 
 void get_simple_tokenizer_module(
   sqlite3_tokenizer_module **ppModule
 ){
   *ppModule = &simpleTokenizerModule;
 }