| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 ** 2007 June 22 | |
| 3 ** | |
| 4 ** The author disclaims copyright to this source code. In place of | |
| 5 ** a legal notice, here is a blessing: | |
| 6 ** | |
| 7 ** May you do good and not evil. | |
| 8 ** May you find forgiveness for yourself and forgive others. | |
| 9 ** May you share freely, never taking more than you give. | |
| 10 ** | |
| 11 ****************************************************************************** | |
| 12 ** | |
| 13 ** This is part of an SQLite module implementing full-text search. | |
| 14 ** This particular file implements the generic tokenizer interface. | |
| 15 */ | |
| 16 | |
| 17 /* | |
| 18 ** The code in this file is only compiled if: | |
| 19 ** | |
| 20 ** * The FTS2 module is being built as an extension | |
| 21 ** (in which case SQLITE_CORE is not defined), or | |
| 22 ** | |
| 23 ** * The FTS2 module is being built into the core of | |
| 24 ** SQLite (in which case SQLITE_ENABLE_FTS2 is defined). | |
| 25 */ | |
| 26 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) | |
| 27 | |
| 28 | |
| 29 #include "sqlite3.h" | |
| 30 #include "sqlite3ext.h" | |
| 31 SQLITE_EXTENSION_INIT3 | |
| 32 | |
| 33 #include "fts2_hash.h" | |
| 34 #include "fts2_tokenizer.h" | |
| 35 #include <assert.h> | |
| 36 | |
| 37 /* | |
| 38 ** Implementation of the SQL scalar function for accessing the underlying | |
| 39 ** hash table. This function may be called as follows: | |
| 40 ** | |
| 41 ** SELECT <function-name>(<key-name>); | |
| 42 ** SELECT <function-name>(<key-name>, <pointer>); | |
| 43 ** | |
| 44 ** where <function-name> is the name passed as the second argument | |
| 45 ** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer'). | |
| 46 ** | |
| 47 ** If the <pointer> argument is specified, it must be a blob value | |
| 48 ** containing a pointer to be stored as the hash data corresponding | |
| 49 ** to the string <key-name>. If <pointer> is not specified, then | |
| 50 ** the string <key-name> must already exist in the has table. Otherwise, | |
| 51 ** an error is returned. | |
| 52 ** | |
| 53 ** Whether or not the <pointer> argument is specified, the value returned | |
| 54 ** is a blob containing the pointer stored as the hash data corresponding | |
| 55 ** to string <key-name> (after the hash-table is updated, if applicable). | |
| 56 */ | |
| 57 static void scalarFunc( | |
| 58 sqlite3_context *context, | |
| 59 int argc, | |
| 60 sqlite3_value **argv | |
| 61 ){ | |
| 62 fts2Hash *pHash; | |
| 63 void *pPtr = 0; | |
| 64 const unsigned char *zName; | |
| 65 int nName; | |
| 66 | |
| 67 assert( argc==1 || argc==2 ); | |
| 68 | |
| 69 pHash = (fts2Hash *)sqlite3_user_data(context); | |
| 70 | |
| 71 zName = sqlite3_value_text(argv[0]); | |
| 72 nName = sqlite3_value_bytes(argv[0])+1; | |
| 73 | |
| 74 if( argc==2 ){ | |
| 75 void *pOld; | |
| 76 int n = sqlite3_value_bytes(argv[1]); | |
| 77 if( n!=sizeof(pPtr) ){ | |
| 78 sqlite3_result_error(context, "argument type mismatch", -1); | |
| 79 return; | |
| 80 } | |
| 81 pPtr = *(void **)sqlite3_value_blob(argv[1]); | |
| 82 pOld = sqlite3Fts2HashInsert(pHash, (void *)zName, nName, pPtr); | |
| 83 if( pOld==pPtr ){ | |
| 84 sqlite3_result_error(context, "out of memory", -1); | |
| 85 return; | |
| 86 } | |
| 87 }else{ | |
| 88 pPtr = sqlite3Fts2HashFind(pHash, zName, nName); | |
| 89 if( !pPtr ){ | |
| 90 char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); | |
| 91 sqlite3_result_error(context, zErr, -1); | |
| 92 sqlite3_free(zErr); | |
| 93 return; | |
| 94 } | |
| 95 } | |
| 96 | |
| 97 sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); | |
| 98 } | |
| 99 | |
| 100 #ifdef SQLITE_TEST | |
| 101 | |
| 102 #include <tcl.h> | |
| 103 #include <string.h> | |
| 104 | |
| 105 /* | |
| 106 ** Implementation of a special SQL scalar function for testing tokenizers | |
| 107 ** designed to be used in concert with the Tcl testing framework. This | |
| 108 ** function must be called with two arguments: | |
| 109 ** | |
| 110 ** SELECT <function-name>(<key-name>, <input-string>); | |
| 111 ** SELECT <function-name>(<key-name>, <pointer>); | |
| 112 ** | |
| 113 ** where <function-name> is the name passed as the second argument | |
| 114 ** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer') | |
| 115 ** concatenated with the string '_test' (e.g. 'fts2_tokenizer_test'). | |
| 116 ** | |
| 117 ** The return value is a string that may be interpreted as a Tcl | |
| 118 ** list. For each token in the <input-string>, three elements are | |
| 119 ** added to the returned list. The first is the token position, the | |
| 120 ** second is the token text (folded, stemmed, etc.) and the third is the | |
| 121 ** substring of <input-string> associated with the token. For example, | |
| 122 ** using the built-in "simple" tokenizer: | |
| 123 ** | |
| 124 ** SELECT fts_tokenizer_test('simple', 'I don't see how'); | |
| 125 ** | |
| 126 ** will return the string: | |
| 127 ** | |
| 128 ** "{0 i I 1 dont don't 2 see see 3 how how}" | |
| 129 ** | |
| 130 */ | |
| 131 static void testFunc( | |
| 132 sqlite3_context *context, | |
| 133 int argc, | |
| 134 sqlite3_value **argv | |
| 135 ){ | |
| 136 fts2Hash *pHash; | |
| 137 sqlite3_tokenizer_module *p; | |
| 138 sqlite3_tokenizer *pTokenizer = 0; | |
| 139 sqlite3_tokenizer_cursor *pCsr = 0; | |
| 140 | |
| 141 const char *zErr = 0; | |
| 142 | |
| 143 const char *zName; | |
| 144 int nName; | |
| 145 const char *zInput; | |
| 146 int nInput; | |
| 147 | |
| 148 const char *zArg = 0; | |
| 149 | |
| 150 const char *zToken; | |
| 151 int nToken; | |
| 152 int iStart; | |
| 153 int iEnd; | |
| 154 int iPos; | |
| 155 | |
| 156 Tcl_Obj *pRet; | |
| 157 | |
| 158 assert( argc==2 || argc==3 ); | |
| 159 | |
| 160 nName = sqlite3_value_bytes(argv[0]); | |
| 161 zName = (const char *)sqlite3_value_text(argv[0]); | |
| 162 nInput = sqlite3_value_bytes(argv[argc-1]); | |
| 163 zInput = (const char *)sqlite3_value_text(argv[argc-1]); | |
| 164 | |
| 165 if( argc==3 ){ | |
| 166 zArg = (const char *)sqlite3_value_text(argv[1]); | |
| 167 } | |
| 168 | |
| 169 pHash = (fts2Hash *)sqlite3_user_data(context); | |
| 170 p = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zName, nName+1); | |
| 171 | |
| 172 if( !p ){ | |
| 173 char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); | |
| 174 sqlite3_result_error(context, zErr, -1); | |
| 175 sqlite3_free(zErr); | |
| 176 return; | |
| 177 } | |
| 178 | |
| 179 pRet = Tcl_NewObj(); | |
| 180 Tcl_IncrRefCount(pRet); | |
| 181 | |
| 182 if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){ | |
| 183 zErr = "error in xCreate()"; | |
| 184 goto finish; | |
| 185 } | |
| 186 pTokenizer->pModule = p; | |
| 187 if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){ | |
| 188 zErr = "error in xOpen()"; | |
| 189 goto finish; | |
| 190 } | |
| 191 pCsr->pTokenizer = pTokenizer; | |
| 192 | |
| 193 while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ | |
| 194 Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); | |
| 195 Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); | |
| 196 zToken = &zInput[iStart]; | |
| 197 nToken = iEnd-iStart; | |
| 198 Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); | |
| 199 } | |
| 200 | |
| 201 if( SQLITE_OK!=p->xClose(pCsr) ){ | |
| 202 zErr = "error in xClose()"; | |
| 203 goto finish; | |
| 204 } | |
| 205 if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ | |
| 206 zErr = "error in xDestroy()"; | |
| 207 goto finish; | |
| 208 } | |
| 209 | |
| 210 finish: | |
| 211 if( zErr ){ | |
| 212 sqlite3_result_error(context, zErr, -1); | |
| 213 }else{ | |
| 214 sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); | |
| 215 } | |
| 216 Tcl_DecrRefCount(pRet); | |
| 217 } | |
| 218 | |
| 219 static | |
| 220 int registerTokenizer( | |
| 221 sqlite3 *db, | |
| 222 char *zName, | |
| 223 const sqlite3_tokenizer_module *p | |
| 224 ){ | |
| 225 int rc; | |
| 226 sqlite3_stmt *pStmt; | |
| 227 const char zSql[] = "SELECT fts2_tokenizer(?, ?)"; | |
| 228 | |
| 229 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); | |
| 230 if( rc!=SQLITE_OK ){ | |
| 231 return rc; | |
| 232 } | |
| 233 | |
| 234 sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); | |
| 235 sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); | |
| 236 sqlite3_step(pStmt); | |
| 237 | |
| 238 return sqlite3_finalize(pStmt); | |
| 239 } | |
| 240 | |
| 241 static | |
| 242 int queryFts2Tokenizer( | |
| 243 sqlite3 *db, | |
| 244 char *zName, | |
| 245 const sqlite3_tokenizer_module **pp | |
| 246 ){ | |
| 247 int rc; | |
| 248 sqlite3_stmt *pStmt; | |
| 249 const char zSql[] = "SELECT fts2_tokenizer(?)"; | |
| 250 | |
| 251 *pp = 0; | |
| 252 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); | |
| 253 if( rc!=SQLITE_OK ){ | |
| 254 return rc; | |
| 255 } | |
| 256 | |
| 257 sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); | |
| 258 if( SQLITE_ROW==sqlite3_step(pStmt) ){ | |
| 259 if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ | |
| 260 memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); | |
| 261 } | |
| 262 } | |
| 263 | |
| 264 return sqlite3_finalize(pStmt); | |
| 265 } | |
| 266 | |
| 267 void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); | |
| 268 | |
| 269 /* | |
| 270 ** Implementation of the scalar function fts2_tokenizer_internal_test(). | |
| 271 ** This function is used for testing only, it is not included in the | |
| 272 ** build unless SQLITE_TEST is defined. | |
| 273 ** | |
| 274 ** The purpose of this is to test that the fts2_tokenizer() function | |
| 275 ** can be used as designed by the C-code in the queryFts2Tokenizer and | |
| 276 ** registerTokenizer() functions above. These two functions are repeated | |
| 277 ** in the README.tokenizer file as an example, so it is important to | |
| 278 ** test them. | |
| 279 ** | |
| 280 ** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar | |
| 281 ** function with no arguments. An assert() will fail if a problem is | |
| 282 ** detected. i.e.: | |
| 283 ** | |
| 284 ** SELECT fts2_tokenizer_internal_test(); | |
| 285 ** | |
| 286 */ | |
| 287 static void intTestFunc( | |
| 288 sqlite3_context *context, | |
| 289 int argc, | |
| 290 sqlite3_value **argv | |
| 291 ){ | |
| 292 int rc; | |
| 293 const sqlite3_tokenizer_module *p1; | |
| 294 const sqlite3_tokenizer_module *p2; | |
| 295 sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); | |
| 296 | |
| 297 /* Test the query function */ | |
| 298 sqlite3Fts2SimpleTokenizerModule(&p1); | |
| 299 rc = queryFts2Tokenizer(db, "simple", &p2); | |
| 300 assert( rc==SQLITE_OK ); | |
| 301 assert( p1==p2 ); | |
| 302 rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2); | |
| 303 assert( rc==SQLITE_ERROR ); | |
| 304 assert( p2==0 ); | |
| 305 assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); | |
| 306 | |
| 307 /* Test the storage function */ | |
| 308 rc = registerTokenizer(db, "nosuchtokenizer", p1); | |
| 309 assert( rc==SQLITE_OK ); | |
| 310 rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2); | |
| 311 assert( rc==SQLITE_OK ); | |
| 312 assert( p2==p1 ); | |
| 313 | |
| 314 sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); | |
| 315 } | |
| 316 | |
| 317 #endif | |
| 318 | |
| 319 /* | |
| 320 ** Set up SQL objects in database db used to access the contents of | |
| 321 ** the hash table pointed to by argument pHash. The hash table must | |
| 322 ** been initialized to use string keys, and to take a private copy | |
| 323 ** of the key when a value is inserted. i.e. by a call similar to: | |
| 324 ** | |
| 325 ** sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1); | |
| 326 ** | |
| 327 ** This function adds a scalar function (see header comment above | |
| 328 ** scalarFunc() in this file for details) and, if ENABLE_TABLE is | |
| 329 ** defined at compilation time, a temporary virtual table (see header | |
| 330 ** comment above struct HashTableVtab) to the database schema. Both | |
| 331 ** provide read/write access to the contents of *pHash. | |
| 332 ** | |
| 333 ** The third argument to this function, zName, is used as the name | |
| 334 ** of both the scalar and, if created, the virtual table. | |
| 335 */ | |
| 336 int sqlite3Fts2InitHashTable( | |
| 337 sqlite3 *db, | |
| 338 fts2Hash *pHash, | |
| 339 const char *zName | |
| 340 ){ | |
| 341 int rc = SQLITE_OK; | |
| 342 void *p = (void *)pHash; | |
| 343 const int any = SQLITE_ANY; | |
| 344 char *zTest = 0; | |
| 345 char *zTest2 = 0; | |
| 346 | |
| 347 #ifdef SQLITE_TEST | |
| 348 void *pdb = (void *)db; | |
| 349 zTest = sqlite3_mprintf("%s_test", zName); | |
| 350 zTest2 = sqlite3_mprintf("%s_internal_test", zName); | |
| 351 if( !zTest || !zTest2 ){ | |
| 352 rc = SQLITE_NOMEM; | |
| 353 } | |
| 354 #endif | |
| 355 | |
| 356 if( rc!=SQLITE_OK | |
| 357 || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0)) | |
| 358 || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0)) | |
| 359 #ifdef SQLITE_TEST | |
| 360 || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0)) | |
| 361 || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0)) | |
| 362 || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0)) | |
| 363 #endif | |
| 364 ); | |
| 365 | |
| 366 sqlite3_free(zTest); | |
| 367 sqlite3_free(zTest2); | |
| 368 return rc; | |
| 369 } | |
| 370 | |
| 371 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */ | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2363173002:
[sqlite] Remove obsolete reference version 3.8.7.4. (Closed)
