Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(363)

Side by Side Diff: third_party/sqlite/src/ext/fts3/fts3_tokenizer.c

Issue 2751253002: [sql] Import SQLite 3.17.0. (Closed)
Patch Set: also clang on Linux i386 Created 3 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 /* 1 /*
2 ** 2007 June 22 2 ** 2007 June 22
3 ** 3 **
4 ** The author disclaims copyright to this source code. In place of 4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing: 5 ** a legal notice, here is a blessing:
6 ** 6 **
7 ** May you do good and not evil. 7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others. 8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give. 9 ** May you share freely, never taking more than you give.
10 ** 10 **
(...skipping 12 matching lines...) Expand all
23 ** * The FTS3 module is being built into the core of 23 ** * The FTS3 module is being built into the core of
24 ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). 24 ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
25 */ 25 */
26 #include "fts3Int.h" 26 #include "fts3Int.h"
27 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) 27 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
28 28
29 #include <assert.h> 29 #include <assert.h>
30 #include <string.h> 30 #include <string.h>
31 31
32 /* 32 /*
33 ** Return true if the two-argument version of fts3_tokenizer()
34 ** has been activated via a prior call to sqlite3_db_config(db,
35 ** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0);
36 */
37 static int fts3TokenizerEnabled(sqlite3_context *context){
38 sqlite3 *db = sqlite3_context_db_handle(context);
39 int isEnabled = 0;
40 sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled);
41 return isEnabled;
42 }
43
44 /*
33 ** Implementation of the SQL scalar function for accessing the underlying 45 ** Implementation of the SQL scalar function for accessing the underlying
34 ** hash table. This function may be called as follows: 46 ** hash table. This function may be called as follows:
35 ** 47 **
36 ** SELECT <function-name>(<key-name>); 48 ** SELECT <function-name>(<key-name>);
37 ** SELECT <function-name>(<key-name>, <pointer>); 49 ** SELECT <function-name>(<key-name>, <pointer>);
38 ** 50 **
39 ** where <function-name> is the name passed as the second argument 51 ** where <function-name> is the name passed as the second argument
40 ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). 52 ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
41 ** 53 **
42 ** If the <pointer> argument is specified, it must be a blob value 54 ** If the <pointer> argument is specified, it must be a blob value
43 ** containing a pointer to be stored as the hash data corresponding 55 ** containing a pointer to be stored as the hash data corresponding
44 ** to the string <key-name>. If <pointer> is not specified, then 56 ** to the string <key-name>. If <pointer> is not specified, then
45 ** the string <key-name> must already exist in the has table. Otherwise, 57 ** the string <key-name> must already exist in the has table. Otherwise,
46 ** an error is returned. 58 ** an error is returned.
47 ** 59 **
48 ** Whether or not the <pointer> argument is specified, the value returned 60 ** Whether or not the <pointer> argument is specified, the value returned
49 ** is a blob containing the pointer stored as the hash data corresponding 61 ** is a blob containing the pointer stored as the hash data corresponding
50 ** to string <key-name> (after the hash-table is updated, if applicable). 62 ** to string <key-name> (after the hash-table is updated, if applicable).
51 */ 63 */
52 static void scalarFunc( 64 static void fts3TokenizerFunc(
53 sqlite3_context *context, 65 sqlite3_context *context,
54 int argc, 66 int argc,
55 sqlite3_value **argv 67 sqlite3_value **argv
56 ){ 68 ){
57 Fts3Hash *pHash; 69 Fts3Hash *pHash;
58 void *pPtr = 0; 70 void *pPtr = 0;
59 const unsigned char *zName; 71 const unsigned char *zName;
60 int nName; 72 int nName;
61 73
62 assert( argc==1 || argc==2 ); 74 assert( argc==1 || argc==2 );
63 75
64 pHash = (Fts3Hash *)sqlite3_user_data(context); 76 pHash = (Fts3Hash *)sqlite3_user_data(context);
65 77
66 zName = sqlite3_value_text(argv[0]); 78 zName = sqlite3_value_text(argv[0]);
67 nName = sqlite3_value_bytes(argv[0])+1; 79 nName = sqlite3_value_bytes(argv[0])+1;
68 80
69 if( argc==2 ){ 81 if( argc==2 ){
70 void *pOld; 82 if( fts3TokenizerEnabled(context) ){
71 int n = sqlite3_value_bytes(argv[1]); 83 void *pOld;
72 if( zName==0 || n!=sizeof(pPtr) ){ 84 int n = sqlite3_value_bytes(argv[1]);
73 sqlite3_result_error(context, "argument type mismatch", -1); 85 if( zName==0 || n!=sizeof(pPtr) ){
74 return; 86 sqlite3_result_error(context, "argument type mismatch", -1);
75 } 87 return;
76 pPtr = *(void **)sqlite3_value_blob(argv[1]); 88 }
77 pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); 89 pPtr = *(void **)sqlite3_value_blob(argv[1]);
78 if( pOld==pPtr ){ 90 pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
79 sqlite3_result_error(context, "out of memory", -1); 91 if( pOld==pPtr ){
92 sqlite3_result_error(context, "out of memory", -1);
93 }
94 }else{
95 sqlite3_result_error(context, "fts3tokenize disabled", -1);
80 return; 96 return;
81 } 97 }
82 }else{ 98 }else{
83 if( zName ){ 99 if( zName ){
84 pPtr = sqlite3Fts3HashFind(pHash, zName, nName); 100 pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
85 } 101 }
86 if( !pPtr ){ 102 if( !pPtr ){
87 char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); 103 char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
88 sqlite3_result_error(context, zErr, -1); 104 sqlite3_result_error(context, zErr, -1);
89 sqlite3_free(zErr); 105 sqlite3_free(zErr);
90 return; 106 return;
91 } 107 }
92 } 108 }
93
94 sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); 109 sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
95 } 110 }
96 111
97 int sqlite3Fts3IsIdChar(char c){ 112 int sqlite3Fts3IsIdChar(char c){
98 static const char isFtsIdChar[] = { 113 static const char isFtsIdChar[] = {
99 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 114 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
100 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ 115 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
101 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ 116 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
102 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ 117 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
103 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ 118 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
(...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after
202 sqlite3_free((void *)aArg); 217 sqlite3_free((void *)aArg);
203 } 218 }
204 219
205 sqlite3_free(zCopy); 220 sqlite3_free(zCopy);
206 return rc; 221 return rc;
207 } 222 }
208 223
209 224
210 #ifdef SQLITE_TEST 225 #ifdef SQLITE_TEST
211 226
212 #include <tcl.h> 227 #if defined(INCLUDE_SQLITE_TCL_H)
228 # include "sqlite_tcl.h"
229 #else
230 # include "tcl.h"
231 #endif
213 #include <string.h> 232 #include <string.h>
214 233
215 /* 234 /*
216 ** Implementation of a special SQL scalar function for testing tokenizers 235 ** Implementation of a special SQL scalar function for testing tokenizers
217 ** designed to be used in concert with the Tcl testing framework. This 236 ** designed to be used in concert with the Tcl testing framework. This
218 ** function must be called with two or more arguments: 237 ** function must be called with two or more arguments:
219 ** 238 **
220 ** SELECT <function-name>(<key-name>, ..., <input-string>); 239 ** SELECT <function-name>(<key-name>, ..., <input-string>);
221 ** 240 **
222 ** where <function-name> is the name passed as the second argument 241 ** where <function-name> is the name passed as the second argument
(...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after
343 return rc; 362 return rc;
344 } 363 }
345 364
346 sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); 365 sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
347 sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); 366 sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
348 sqlite3_step(pStmt); 367 sqlite3_step(pStmt);
349 368
350 return sqlite3_finalize(pStmt); 369 return sqlite3_finalize(pStmt);
351 } 370 }
352 371
372
353 static 373 static
354 int queryTokenizer( 374 int queryTokenizer(
355 sqlite3 *db, 375 sqlite3 *db,
356 char *zName, 376 char *zName,
357 const sqlite3_tokenizer_module **pp 377 const sqlite3_tokenizer_module **pp
358 ){ 378 ){
359 int rc; 379 int rc;
360 sqlite3_stmt *pStmt; 380 sqlite3_stmt *pStmt;
361 const char zSql[] = "SELECT fts3_tokenizer(?)"; 381 const char zSql[] = "SELECT fts3_tokenizer(?)";
362 382
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after
413 sqlite3Fts3SimpleTokenizerModule(&p1); 433 sqlite3Fts3SimpleTokenizerModule(&p1);
414 rc = queryTokenizer(db, "simple", &p2); 434 rc = queryTokenizer(db, "simple", &p2);
415 assert( rc==SQLITE_OK ); 435 assert( rc==SQLITE_OK );
416 assert( p1==p2 ); 436 assert( p1==p2 );
417 rc = queryTokenizer(db, "nosuchtokenizer", &p2); 437 rc = queryTokenizer(db, "nosuchtokenizer", &p2);
418 assert( rc==SQLITE_ERROR ); 438 assert( rc==SQLITE_ERROR );
419 assert( p2==0 ); 439 assert( p2==0 );
420 assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); 440 assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
421 441
422 /* Test the storage function */ 442 /* Test the storage function */
423 rc = registerTokenizer(db, "nosuchtokenizer", p1); 443 if( fts3TokenizerEnabled(context) ){
424 assert( rc==SQLITE_OK ); 444 rc = registerTokenizer(db, "nosuchtokenizer", p1);
425 rc = queryTokenizer(db, "nosuchtokenizer", &p2); 445 assert( rc==SQLITE_OK );
426 assert( rc==SQLITE_OK ); 446 rc = queryTokenizer(db, "nosuchtokenizer", &p2);
427 assert( p2==p1 ); 447 assert( rc==SQLITE_OK );
448 assert( p2==p1 );
449 }
428 450
429 sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); 451 sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
430 } 452 }
431 453
432 #endif 454 #endif
433 455
434 /* 456 /*
435 ** Set up SQL objects in database db used to access the contents of 457 ** Set up SQL objects in database db used to access the contents of
436 ** the hash table pointed to by argument pHash. The hash table must 458 ** the hash table pointed to by argument pHash. The hash table must
437 ** been initialized to use string keys, and to take a private copy 459 ** been initialized to use string keys, and to take a private copy
438 ** of the key when a value is inserted. i.e. by a call similar to: 460 ** of the key when a value is inserted. i.e. by a call similar to:
439 ** 461 **
440 ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); 462 ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
441 ** 463 **
442 ** This function adds a scalar function (see header comment above 464 ** This function adds a scalar function (see header comment above
443 ** scalarFunc() in this file for details) and, if ENABLE_TABLE is 465 ** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is
444 ** defined at compilation time, a temporary virtual table (see header 466 ** defined at compilation time, a temporary virtual table (see header
445 ** comment above struct HashTableVtab) to the database schema. Both 467 ** comment above struct HashTableVtab) to the database schema. Both
446 ** provide read/write access to the contents of *pHash. 468 ** provide read/write access to the contents of *pHash.
447 ** 469 **
448 ** The third argument to this function, zName, is used as the name 470 ** The third argument to this function, zName, is used as the name
449 ** of both the scalar and, if created, the virtual table. 471 ** of both the scalar and, if created, the virtual table.
450 */ 472 */
451 int sqlite3Fts3InitHashTable( 473 int sqlite3Fts3InitHashTable(
452 sqlite3 *db, 474 sqlite3 *db,
453 Fts3Hash *pHash, 475 Fts3Hash *pHash,
454 const char *zName 476 const char *zName
455 ){ 477 ){
456 int rc = SQLITE_OK; 478 int rc = SQLITE_OK;
457 void *p = (void *)pHash; 479 void *p = (void *)pHash;
458 const int any = SQLITE_ANY; 480 const int any = SQLITE_ANY;
459 481
460 #ifdef SQLITE_TEST 482 #ifdef SQLITE_TEST
461 char *zTest = 0; 483 char *zTest = 0;
462 char *zTest2 = 0; 484 char *zTest2 = 0;
463 void *pdb = (void *)db; 485 void *pdb = (void *)db;
464 zTest = sqlite3_mprintf("%s_test", zName); 486 zTest = sqlite3_mprintf("%s_test", zName);
465 zTest2 = sqlite3_mprintf("%s_internal_test", zName); 487 zTest2 = sqlite3_mprintf("%s_internal_test", zName);
466 if( !zTest || !zTest2 ){ 488 if( !zTest || !zTest2 ){
467 rc = SQLITE_NOMEM; 489 rc = SQLITE_NOMEM;
468 } 490 }
469 #endif 491 #endif
470 492
471 if( SQLITE_OK==rc ){ 493 if( SQLITE_OK==rc ){
472 rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); 494 rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0);
473 } 495 }
474 if( SQLITE_OK==rc ){ 496 if( SQLITE_OK==rc ){
475 rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); 497 rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0);
476 } 498 }
477 #ifdef SQLITE_TEST 499 #ifdef SQLITE_TEST
478 if( SQLITE_OK==rc ){ 500 if( SQLITE_OK==rc ){
479 rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); 501 rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0);
480 } 502 }
481 if( SQLITE_OK==rc ){ 503 if( SQLITE_OK==rc ){
482 rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); 504 rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0);
483 } 505 }
484 #endif 506 #endif
485 507
486 #ifdef SQLITE_TEST 508 #ifdef SQLITE_TEST
487 sqlite3_free(zTest); 509 sqlite3_free(zTest);
488 sqlite3_free(zTest2); 510 sqlite3_free(zTest2);
489 #endif 511 #endif
490 512
491 return rc; 513 return rc;
492 } 514 }
493 515
494 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ 516 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
OLDNEW
« no previous file with comments | « third_party/sqlite/src/ext/fts3/fts3_test.c ('k') | third_party/sqlite/src/ext/fts3/fts3_write.c » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698