Cleanup SQLite 3.6.18 import.

third_party/sqlite/src/ext/fts1/fts1_tokenizer1.c

 /*
 ** The author disclaims copyright to this source code.
 **
 *************************************************************************
 ** Implementation of the "simple" full-text-search tokenizer.
 */
 
 /*
 ** The code in this file is only compiled if:
 **
@@ skipping 164 matching lines @@
       if( n>c->nTokenAllocated ){
         c->nTokenAllocated = n+20;
         c->pToken = realloc(c->pToken, c->nTokenAllocated);
         if( c->pToken==NULL ) return SQLITE_NOMEM;
       }
       for(i=0; i<n; i++){
         /* TODO(shess) This needs expansion to handle UTF-8
         ** case-insensitivity.
         */
         unsigned char ch = p[iStartOffset+i];
-        c->pToken[i] = (ch>='A' && ch<='Z') ? (ch-'A'+'a') : ch;
+        c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
       }
       *ppToken = c->pToken;
       *pnBytes = n;
       *piStartOffset = iStartOffset;
       *piEndOffset = c->iOffset;
       *piPosition = c->iToken++;
 
       return SQLITE_OK;
     }
   }
@@ skipping 16 matching lines @@
 ** Allocate a new simple tokenizer.  Return a pointer to the new
 ** tokenizer in *ppModule
 */
 void sqlite3Fts1SimpleTokenizerModule(
   sqlite3_tokenizer_module const**ppModule
 ){
   *ppModule = &simpleTokenizerModule;
 }
 
 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */