| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 ** 2007 June 22 | |
| 3 ** | |
| 4 ** The author disclaims copyright to this source code. In place of | |
| 5 ** a legal notice, here is a blessing: | |
| 6 ** | |
| 7 ** May you do good and not evil. | |
| 8 ** May you find forgiveness for yourself and forgive others. | |
| 9 ** May you share freely, never taking more than you give. | |
| 10 ** | |
| 11 ****************************************************************************** | |
| 12 ** | |
| 13 ** This is part of an SQLite module implementing full-text search. | |
| 14 ** This particular file implements the generic tokenizer interface. | |
| 15 */ | |
| 16 | |
| 17 /* | |
| 18 ** The code in this file is only compiled if: | |
| 19 ** | |
| 20 ** * The FTS3 module is being built as an extension | |
| 21 ** (in which case SQLITE_CORE is not defined), or | |
| 22 ** | |
| 23 ** * The FTS3 module is being built into the core of | |
| 24 ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). | |
| 25 */ | |
| 26 #include "fts3Int.h" | |
| 27 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) | |
| 28 | |
| 29 #include <assert.h> | |
| 30 #include <string.h> | |
| 31 | |
| 32 /* | |
| 33 ** Implementation of the SQL scalar function for accessing the underlying | |
| 34 ** hash table. This function may be called as follows: | |
| 35 ** | |
| 36 ** SELECT <function-name>(<key-name>); | |
| 37 ** SELECT <function-name>(<key-name>, <pointer>); | |
| 38 ** | |
| 39 ** where <function-name> is the name passed as the second argument | |
| 40 ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). | |
| 41 ** | |
| 42 ** If the <pointer> argument is specified, it must be a blob value | |
| 43 ** containing a pointer to be stored as the hash data corresponding | |
| 44 ** to the string <key-name>. If <pointer> is not specified, then | |
| 45 ** the string <key-name> must already exist in the has table. Otherwise, | |
| 46 ** an error is returned. | |
| 47 ** | |
| 48 ** Whether or not the <pointer> argument is specified, the value returned | |
| 49 ** is a blob containing the pointer stored as the hash data corresponding | |
| 50 ** to string <key-name> (after the hash-table is updated, if applicable). | |
| 51 */ | |
| 52 static void scalarFunc( | |
| 53 sqlite3_context *context, | |
| 54 int argc, | |
| 55 sqlite3_value **argv | |
| 56 ){ | |
| 57 Fts3Hash *pHash; | |
| 58 void *pPtr = 0; | |
| 59 const unsigned char *zName; | |
| 60 int nName; | |
| 61 | |
| 62 assert( argc==1 || argc==2 ); | |
| 63 | |
| 64 pHash = (Fts3Hash *)sqlite3_user_data(context); | |
| 65 | |
| 66 zName = sqlite3_value_text(argv[0]); | |
| 67 nName = sqlite3_value_bytes(argv[0])+1; | |
| 68 | |
| 69 if( argc==2 ){ | |
| 70 void *pOld; | |
| 71 int n = sqlite3_value_bytes(argv[1]); | |
| 72 if( n!=sizeof(pPtr) ){ | |
| 73 sqlite3_result_error(context, "argument type mismatch", -1); | |
| 74 return; | |
| 75 } | |
| 76 pPtr = *(void **)sqlite3_value_blob(argv[1]); | |
| 77 pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); | |
| 78 if( pOld==pPtr ){ | |
| 79 sqlite3_result_error(context, "out of memory", -1); | |
| 80 return; | |
| 81 } | |
| 82 }else{ | |
| 83 pPtr = sqlite3Fts3HashFind(pHash, zName, nName); | |
| 84 if( !pPtr ){ | |
| 85 char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); | |
| 86 sqlite3_result_error(context, zErr, -1); | |
| 87 sqlite3_free(zErr); | |
| 88 return; | |
| 89 } | |
| 90 } | |
| 91 | |
| 92 sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); | |
| 93 } | |
| 94 | |
| 95 int sqlite3Fts3IsIdChar(char c){ | |
| 96 static const char isFtsIdChar[] = { | |
| 97 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ | |
| 98 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ | |
| 99 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ | |
| 100 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ | |
| 101 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ | |
| 102 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ | |
| 103 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ | |
| 104 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ | |
| 105 }; | |
| 106 return (c&0x80 || isFtsIdChar[(int)(c)]); | |
| 107 } | |
| 108 | |
| 109 const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ | |
| 110 const char *z1; | |
| 111 const char *z2 = 0; | |
| 112 | |
| 113 /* Find the start of the next token. */ | |
| 114 z1 = zStr; | |
| 115 while( z2==0 ){ | |
| 116 char c = *z1; | |
| 117 switch( c ){ | |
| 118 case '\0': return 0; /* No more tokens here */ | |
| 119 case '\'': | |
| 120 case '"': | |
| 121 case '`': { | |
| 122 z2 = z1; | |
| 123 while( *++z2 && (*z2!=c || *++z2==c) ); | |
| 124 break; | |
| 125 } | |
| 126 case '[': | |
| 127 z2 = &z1[1]; | |
| 128 while( *z2 && z2[0]!=']' ) z2++; | |
| 129 if( *z2 ) z2++; | |
| 130 break; | |
| 131 | |
| 132 default: | |
| 133 if( sqlite3Fts3IsIdChar(*z1) ){ | |
| 134 z2 = &z1[1]; | |
| 135 while( sqlite3Fts3IsIdChar(*z2) ) z2++; | |
| 136 }else{ | |
| 137 z1++; | |
| 138 } | |
| 139 } | |
| 140 } | |
| 141 | |
| 142 *pn = (int)(z2-z1); | |
| 143 return z1; | |
| 144 } | |
| 145 | |
| 146 int sqlite3Fts3InitTokenizer( | |
| 147 Fts3Hash *pHash, /* Tokenizer hash table */ | |
| 148 const char *zArg, /* Tokenizer name */ | |
| 149 sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ | |
| 150 char **pzErr /* OUT: Set to malloced error message */ | |
| 151 ){ | |
| 152 int rc; | |
| 153 char *z = (char *)zArg; | |
| 154 int n = 0; | |
| 155 char *zCopy; | |
| 156 char *zEnd; /* Pointer to nul-term of zCopy */ | |
| 157 sqlite3_tokenizer_module *m; | |
| 158 | |
| 159 zCopy = sqlite3_mprintf("%s", zArg); | |
| 160 if( !zCopy ) return SQLITE_NOMEM; | |
| 161 zEnd = &zCopy[strlen(zCopy)]; | |
| 162 | |
| 163 z = (char *)sqlite3Fts3NextToken(zCopy, &n); | |
| 164 z[n] = '\0'; | |
| 165 sqlite3Fts3Dequote(z); | |
| 166 | |
| 167 m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); | |
| 168 if( !m ){ | |
| 169 *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); | |
| 170 rc = SQLITE_ERROR; | |
| 171 }else{ | |
| 172 char const **aArg = 0; | |
| 173 int iArg = 0; | |
| 174 z = &z[n+1]; | |
| 175 while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){ | |
| 176 int nNew = sizeof(char *)*(iArg+1); | |
| 177 char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew); | |
| 178 if( !aNew ){ | |
| 179 sqlite3_free(zCopy); | |
| 180 sqlite3_free((void *)aArg); | |
| 181 return SQLITE_NOMEM; | |
| 182 } | |
| 183 aArg = aNew; | |
| 184 aArg[iArg++] = z; | |
| 185 z[n] = '\0'; | |
| 186 sqlite3Fts3Dequote(z); | |
| 187 z = &z[n+1]; | |
| 188 } | |
| 189 rc = m->xCreate(iArg, aArg, ppTok); | |
| 190 assert( rc!=SQLITE_OK || *ppTok ); | |
| 191 if( rc!=SQLITE_OK ){ | |
| 192 *pzErr = sqlite3_mprintf("unknown tokenizer"); | |
| 193 }else{ | |
| 194 (*ppTok)->pModule = m; | |
| 195 } | |
| 196 sqlite3_free((void *)aArg); | |
| 197 } | |
| 198 | |
| 199 sqlite3_free(zCopy); | |
| 200 return rc; | |
| 201 } | |
| 202 | |
| 203 | |
| 204 #ifdef SQLITE_TEST | |
| 205 | |
| 206 #include <tcl.h> | |
| 207 #include <string.h> | |
| 208 | |
| 209 /* | |
| 210 ** Implementation of a special SQL scalar function for testing tokenizers | |
| 211 ** designed to be used in concert with the Tcl testing framework. This | |
| 212 ** function must be called with two or more arguments: | |
| 213 ** | |
| 214 ** SELECT <function-name>(<key-name>, ..., <input-string>); | |
| 215 ** | |
| 216 ** where <function-name> is the name passed as the second argument | |
| 217 ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') | |
| 218 ** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). | |
| 219 ** | |
| 220 ** The return value is a string that may be interpreted as a Tcl | |
| 221 ** list. For each token in the <input-string>, three elements are | |
| 222 ** added to the returned list. The first is the token position, the | |
| 223 ** second is the token text (folded, stemmed, etc.) and the third is the | |
| 224 ** substring of <input-string> associated with the token. For example, | |
| 225 ** using the built-in "simple" tokenizer: | |
| 226 ** | |
| 227 ** SELECT fts_tokenizer_test('simple', 'I don't see how'); | |
| 228 ** | |
| 229 ** will return the string: | |
| 230 ** | |
| 231 ** "{0 i I 1 dont don't 2 see see 3 how how}" | |
| 232 ** | |
| 233 */ | |
| 234 static void testFunc( | |
| 235 sqlite3_context *context, | |
| 236 int argc, | |
| 237 sqlite3_value **argv | |
| 238 ){ | |
| 239 Fts3Hash *pHash; | |
| 240 sqlite3_tokenizer_module *p; | |
| 241 sqlite3_tokenizer *pTokenizer = 0; | |
| 242 sqlite3_tokenizer_cursor *pCsr = 0; | |
| 243 | |
| 244 const char *zErr = 0; | |
| 245 | |
| 246 const char *zName; | |
| 247 int nName; | |
| 248 const char *zInput; | |
| 249 int nInput; | |
| 250 | |
| 251 const char *azArg[64]; | |
| 252 | |
| 253 const char *zToken; | |
| 254 int nToken = 0; | |
| 255 int iStart = 0; | |
| 256 int iEnd = 0; | |
| 257 int iPos = 0; | |
| 258 int i; | |
| 259 | |
| 260 Tcl_Obj *pRet; | |
| 261 | |
| 262 if( argc<2 ){ | |
| 263 sqlite3_result_error(context, "insufficient arguments", -1); | |
| 264 return; | |
| 265 } | |
| 266 | |
| 267 nName = sqlite3_value_bytes(argv[0]); | |
| 268 zName = (const char *)sqlite3_value_text(argv[0]); | |
| 269 nInput = sqlite3_value_bytes(argv[argc-1]); | |
| 270 zInput = (const char *)sqlite3_value_text(argv[argc-1]); | |
| 271 | |
| 272 pHash = (Fts3Hash *)sqlite3_user_data(context); | |
| 273 p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); | |
| 274 | |
| 275 if( !p ){ | |
| 276 char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); | |
| 277 sqlite3_result_error(context, zErr, -1); | |
| 278 sqlite3_free(zErr); | |
| 279 return; | |
| 280 } | |
| 281 | |
| 282 pRet = Tcl_NewObj(); | |
| 283 Tcl_IncrRefCount(pRet); | |
| 284 | |
| 285 for(i=1; i<argc-1; i++){ | |
| 286 azArg[i-1] = (const char *)sqlite3_value_text(argv[i]); | |
| 287 } | |
| 288 | |
| 289 if( SQLITE_OK!=p->xCreate(argc-2, azArg, &pTokenizer) ){ | |
| 290 zErr = "error in xCreate()"; | |
| 291 goto finish; | |
| 292 } | |
| 293 pTokenizer->pModule = p; | |
| 294 if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ | |
| 295 zErr = "error in xOpen()"; | |
| 296 goto finish; | |
| 297 } | |
| 298 | |
| 299 while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ | |
| 300 Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); | |
| 301 Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); | |
| 302 zToken = &zInput[iStart]; | |
| 303 nToken = iEnd-iStart; | |
| 304 Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); | |
| 305 } | |
| 306 | |
| 307 if( SQLITE_OK!=p->xClose(pCsr) ){ | |
| 308 zErr = "error in xClose()"; | |
| 309 goto finish; | |
| 310 } | |
| 311 if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ | |
| 312 zErr = "error in xDestroy()"; | |
| 313 goto finish; | |
| 314 } | |
| 315 | |
| 316 finish: | |
| 317 if( zErr ){ | |
| 318 sqlite3_result_error(context, zErr, -1); | |
| 319 }else{ | |
| 320 sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); | |
| 321 } | |
| 322 Tcl_DecrRefCount(pRet); | |
| 323 } | |
| 324 | |
| 325 static | |
| 326 int registerTokenizer( | |
| 327 sqlite3 *db, | |
| 328 char *zName, | |
| 329 const sqlite3_tokenizer_module *p | |
| 330 ){ | |
| 331 int rc; | |
| 332 sqlite3_stmt *pStmt; | |
| 333 const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; | |
| 334 | |
| 335 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); | |
| 336 if( rc!=SQLITE_OK ){ | |
| 337 return rc; | |
| 338 } | |
| 339 | |
| 340 sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); | |
| 341 sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); | |
| 342 sqlite3_step(pStmt); | |
| 343 | |
| 344 return sqlite3_finalize(pStmt); | |
| 345 } | |
| 346 | |
| 347 static | |
| 348 int queryTokenizer( | |
| 349 sqlite3 *db, | |
| 350 char *zName, | |
| 351 const sqlite3_tokenizer_module **pp | |
| 352 ){ | |
| 353 int rc; | |
| 354 sqlite3_stmt *pStmt; | |
| 355 const char zSql[] = "SELECT fts3_tokenizer(?)"; | |
| 356 | |
| 357 *pp = 0; | |
| 358 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); | |
| 359 if( rc!=SQLITE_OK ){ | |
| 360 return rc; | |
| 361 } | |
| 362 | |
| 363 sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); | |
| 364 if( SQLITE_ROW==sqlite3_step(pStmt) ){ | |
| 365 if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ | |
| 366 memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); | |
| 367 } | |
| 368 } | |
| 369 | |
| 370 return sqlite3_finalize(pStmt); | |
| 371 } | |
| 372 | |
| 373 void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); | |
| 374 | |
| 375 /* | |
| 376 ** Implementation of the scalar function fts3_tokenizer_internal_test(). | |
| 377 ** This function is used for testing only, it is not included in the | |
| 378 ** build unless SQLITE_TEST is defined. | |
| 379 ** | |
| 380 ** The purpose of this is to test that the fts3_tokenizer() function | |
| 381 ** can be used as designed by the C-code in the queryTokenizer and | |
| 382 ** registerTokenizer() functions above. These two functions are repeated | |
| 383 ** in the README.tokenizer file as an example, so it is important to | |
| 384 ** test them. | |
| 385 ** | |
| 386 ** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar | |
| 387 ** function with no arguments. An assert() will fail if a problem is | |
| 388 ** detected. i.e.: | |
| 389 ** | |
| 390 ** SELECT fts3_tokenizer_internal_test(); | |
| 391 ** | |
| 392 */ | |
| 393 static void intTestFunc( | |
| 394 sqlite3_context *context, | |
| 395 int argc, | |
| 396 sqlite3_value **argv | |
| 397 ){ | |
| 398 int rc; | |
| 399 const sqlite3_tokenizer_module *p1; | |
| 400 const sqlite3_tokenizer_module *p2; | |
| 401 sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); | |
| 402 | |
| 403 UNUSED_PARAMETER(argc); | |
| 404 UNUSED_PARAMETER(argv); | |
| 405 | |
| 406 /* Test the query function */ | |
| 407 sqlite3Fts3SimpleTokenizerModule(&p1); | |
| 408 rc = queryTokenizer(db, "simple", &p2); | |
| 409 assert( rc==SQLITE_OK ); | |
| 410 assert( p1==p2 ); | |
| 411 rc = queryTokenizer(db, "nosuchtokenizer", &p2); | |
| 412 assert( rc==SQLITE_ERROR ); | |
| 413 assert( p2==0 ); | |
| 414 assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); | |
| 415 | |
| 416 /* Test the storage function */ | |
| 417 rc = registerTokenizer(db, "nosuchtokenizer", p1); | |
| 418 assert( rc==SQLITE_OK ); | |
| 419 rc = queryTokenizer(db, "nosuchtokenizer", &p2); | |
| 420 assert( rc==SQLITE_OK ); | |
| 421 assert( p2==p1 ); | |
| 422 | |
| 423 sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); | |
| 424 } | |
| 425 | |
| 426 #endif | |
| 427 | |
| 428 /* | |
| 429 ** Set up SQL objects in database db used to access the contents of | |
| 430 ** the hash table pointed to by argument pHash. The hash table must | |
| 431 ** been initialized to use string keys, and to take a private copy | |
| 432 ** of the key when a value is inserted. i.e. by a call similar to: | |
| 433 ** | |
| 434 ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); | |
| 435 ** | |
| 436 ** This function adds a scalar function (see header comment above | |
| 437 ** scalarFunc() in this file for details) and, if ENABLE_TABLE is | |
| 438 ** defined at compilation time, a temporary virtual table (see header | |
| 439 ** comment above struct HashTableVtab) to the database schema. Both | |
| 440 ** provide read/write access to the contents of *pHash. | |
| 441 ** | |
| 442 ** The third argument to this function, zName, is used as the name | |
| 443 ** of both the scalar and, if created, the virtual table. | |
| 444 */ | |
| 445 int sqlite3Fts3InitHashTable( | |
| 446 sqlite3 *db, | |
| 447 Fts3Hash *pHash, | |
| 448 const char *zName | |
| 449 ){ | |
| 450 int rc = SQLITE_OK; | |
| 451 void *p = (void *)pHash; | |
| 452 const int any = SQLITE_ANY; | |
| 453 | |
| 454 #ifdef SQLITE_TEST | |
| 455 char *zTest = 0; | |
| 456 char *zTest2 = 0; | |
| 457 void *pdb = (void *)db; | |
| 458 zTest = sqlite3_mprintf("%s_test", zName); | |
| 459 zTest2 = sqlite3_mprintf("%s_internal_test", zName); | |
| 460 if( !zTest || !zTest2 ){ | |
| 461 rc = SQLITE_NOMEM; | |
| 462 } | |
| 463 #endif | |
| 464 | |
| 465 if( SQLITE_OK==rc ){ | |
| 466 rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); | |
| 467 } | |
| 468 if( SQLITE_OK==rc ){ | |
| 469 rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); | |
| 470 } | |
| 471 #ifdef SQLITE_TEST | |
| 472 if( SQLITE_OK==rc ){ | |
| 473 rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); | |
| 474 } | |
| 475 if( SQLITE_OK==rc ){ | |
| 476 rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); | |
| 477 } | |
| 478 #endif | |
| 479 | |
| 480 #ifdef SQLITE_TEST | |
| 481 sqlite3_free(zTest); | |
| 482 sqlite3_free(zTest2); | |
| 483 #endif | |
| 484 | |
| 485 return rc; | |
| 486 } | |
| 487 | |
| 488 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2363173002:
[sqlite] Remove obsolete reference version 3.8.7.4. (Closed)
