| OLD | NEW |
| 1 # 2009 December 03 | 1 # 2009 December 03 |
| 2 # | 2 # |
| 3 # May you do good and not evil. | 3 # May you do good and not evil. |
| 4 # May you find forgiveness for yourself and forgive others. | 4 # May you find forgiveness for yourself and forgive others. |
| 5 # May you share freely, never taking more than you give. | 5 # May you share freely, never taking more than you give. |
| 6 # | 6 # |
| 7 #*********************************************************************** | 7 #*********************************************************************** |
| 8 # | 8 # |
| 9 # Brute force (random data) tests for FTS3. | 9 # Brute force (random data) tests for FTS3. |
| 10 # | 10 # |
| (...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 155 if {[regexp $reg " $col "]} { lappend ret $key ; break } | 155 if {[regexp $reg " $col "]} { lappend ret $key ; break } |
| 156 } | 156 } |
| 157 } | 157 } |
| 158 | 158 |
| 159 #lsort -uniq -integer $ret | 159 #lsort -uniq -integer $ret |
| 160 set ret | 160 set ret |
| 161 } | 161 } |
| 162 | 162 |
| 163 # This [proc] is used to test the FTS3 matchinfo() function. | 163 # This [proc] is used to test the FTS3 matchinfo() function. |
| 164 # | 164 # |
| 165 proc simple_token_matchinfo {zToken} { | 165 proc simple_token_matchinfo {zToken bDesc} { |
| 166 | 166 |
| 167 set nDoc(0) 0 | 167 set nDoc(0) 0 |
| 168 set nDoc(1) 0 | 168 set nDoc(1) 0 |
| 169 set nDoc(2) 0 | 169 set nDoc(2) 0 |
| 170 set nHit(0) 0 | 170 set nHit(0) 0 |
| 171 set nHit(1) 0 | 171 set nHit(1) 0 |
| 172 set nHit(2) 0 | 172 set nHit(2) 0 |
| 173 | 173 |
| 174 set dir -inc |
| 175 if {$bDesc} { set dir -dec } |
| 174 | 176 |
| 175 foreach key [array names ::t1] { | 177 foreach key [array names ::t1] { |
| 176 set value $::t1($key) | 178 set value $::t1($key) |
| 177 set a($key) [list] | 179 set a($key) [list] |
| 178 foreach i {0 1 2} col $value { | 180 foreach i {0 1 2} col $value { |
| 179 set hit [llength [lsearch -all $col $zToken]] | 181 set hit [llength [lsearch -all $col $zToken]] |
| 180 lappend a($key) $hit | 182 lappend a($key) $hit |
| 181 incr nHit($i) $hit | 183 incr nHit($i) $hit |
| 182 if {$hit>0} { incr nDoc($i) } | 184 if {$hit>0} { incr nDoc($i) } |
| 183 } | 185 } |
| 184 } | 186 } |
| 185 | 187 |
| 186 set ret [list] | 188 set ret [list] |
| 187 foreach docid [lsort -integer [array names a]] { | 189 foreach docid [lsort -integer $dir [array names a]] { |
| 188 if { [lindex [lsort -integer $a($docid)] end] } { | 190 if { [lindex [lsort -integer $a($docid)] end] } { |
| 189 set matchinfo [list 1 3] | 191 set matchinfo [list 1 3] |
| 190 foreach i {0 1 2} hit $a($docid) { | 192 foreach i {0 1 2} hit $a($docid) { |
| 191 lappend matchinfo $hit $nHit($i) $nDoc($i) | 193 lappend matchinfo $hit $nHit($i) $nDoc($i) |
| 192 } | 194 } |
| 193 lappend ret $docid $matchinfo | 195 lappend ret $docid $matchinfo |
| 194 } | 196 } |
| 195 } | 197 } |
| 196 | 198 |
| 197 set ret | 199 set ret |
| (...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 255 return $ret | 257 return $ret |
| 256 } | 258 } |
| 257 | 259 |
| 258 proc mit {blob} { | 260 proc mit {blob} { |
| 259 set scan(littleEndian) i* | 261 set scan(littleEndian) i* |
| 260 set scan(bigEndian) I* | 262 set scan(bigEndian) I* |
| 261 binary scan $blob $scan($::tcl_platform(byteOrder)) r | 263 binary scan $blob $scan($::tcl_platform(byteOrder)) r |
| 262 return $r | 264 return $r |
| 263 } | 265 } |
| 264 db func mit mit | 266 db func mit mit |
| 265 | |
| 266 set sqlite_fts3_enable_parentheses 1 | 267 set sqlite_fts3_enable_parentheses 1 |
| 267 | 268 |
| 268 foreach nodesize {50 500 1000 2000} { | 269 proc do_orderbydocid_test {tn sql res} { |
| 270 uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res] |
| 271 uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \ |
| 272 [lsort -int -dec $res] |
| 273 ] |
| 274 } |
| 275 |
| 276 set NUM_TRIALS 100 |
| 277 |
| 278 foreach {nodesize order} { |
| 279 50 DESC |
| 280 50 ASC |
| 281 500 ASC |
| 282 1000 DESC |
| 283 2000 ASC |
| 284 } { |
| 269 catch { array unset ::t1 } | 285 catch { array unset ::t1 } |
| 286 set testname "$nodesize/$order" |
| 270 | 287 |
| 271 # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. | 288 # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. |
| 272 # | 289 # |
| 273 db transaction { | 290 db transaction { |
| 274 catchsql { DROP TABLE t1 } | 291 catchsql { DROP TABLE t1 } |
| 275 execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)" | 292 execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)" |
| 276 execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')" | 293 execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')" |
| 277 for {set i 0} {$i < 100} {incr i} { insert_row $i } | 294 for {set i 0} {$i < 100} {incr i} { insert_row $i } |
| 278 } | 295 } |
| 279 | 296 |
| 280 for {set iTest 1} {$iTest <= 100} {incr iTest} { | 297 for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} { |
| 281 catchsql COMMIT | 298 catchsql COMMIT |
| 282 | 299 |
| 283 set DO_MALLOC_TEST 0 | 300 set DO_MALLOC_TEST 0 |
| 284 set nRep 10 | 301 set nRep 10 |
| 285 if {$iTest==100 && $nodesize==50} { | 302 if {$iTest==100 && $nodesize==50} { |
| 286 set DO_MALLOC_TEST 1 | 303 set DO_MALLOC_TEST 1 |
| 287 set nRep 2 | 304 set nRep 2 |
| 288 } | 305 } |
| 306 |
| 307 set ::testprefix fts3rnd-1.$testname.$iTest |
| 289 | 308 |
| 290 # Delete one row, update one row and insert one row. | 309 # Delete one row, update one row and insert one row. |
| 291 # | 310 # |
| 292 set rows [array names ::t1] | 311 set rows [array names ::t1] |
| 293 set nRow [llength $rows] | 312 set nRow [llength $rows] |
| 294 set iUpdate [lindex $rows [expr {int(rand()*$nRow)}]] | 313 set iUpdate [lindex $rows [expr {int(rand()*$nRow)}]] |
| 295 set iDelete $iUpdate | 314 set iDelete $iUpdate |
| 296 while {$iDelete == $iUpdate} { | 315 while {$iDelete == $iUpdate} { |
| 297 set iDelete [lindex $rows [expr {int(rand()*$nRow)}]] | 316 set iDelete [lindex $rows [expr {int(rand()*$nRow)}]] |
| 298 } | 317 } |
| 299 set iInsert $iUpdate | 318 set iInsert $iUpdate |
| 300 while {[info exists ::t1($iInsert)]} { | 319 while {[info exists ::t1($iInsert)]} { |
| 301 set iInsert [expr {int(rand()*1000000)}] | 320 set iInsert [expr {int(rand()*1000000)}] |
| 302 } | 321 } |
| 303 execsql BEGIN | 322 execsql BEGIN |
| 304 insert_row $iInsert | 323 insert_row $iInsert |
| 305 update_row $iUpdate | 324 update_row $iUpdate |
| 306 delete_row $iDelete | 325 delete_row $iDelete |
| 307 if {0==($iTest%2)} { execsql COMMIT } | 326 if {0==($iTest%2)} { execsql COMMIT } |
| 308 | 327 |
| 309 if {0==($iTest%2)} { | 328 if {0==($iTest%2)} { |
| 310 do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok | 329 #do_test 0 { fts3_integrity_check t1 } ok |
| 311 } | 330 } |
| 312 | 331 |
| 313 # Pick 10 terms from the vocabulary. Check that the results of querying | 332 # Pick 10 terms from the vocabulary. Check that the results of querying |
| 314 # the database for the set of documents containing each of these terms | 333 # the database for the set of documents containing each of these terms |
| 315 # is the same as the result obtained by scanning the contents of the Tcl | 334 # is the same as the result obtained by scanning the contents of the Tcl |
| 316 # array for each term. | 335 # array for each term. |
| 317 # | 336 # |
| 318 for {set i 0} {$i < 10} {incr i} { | 337 for {set i 0} {$i < 10} {incr i} { |
| 319 set term [random_term] | 338 set term [random_term] |
| 320 do_select_test fts3rnd-1.$nodesize.$iTest.1.$i { | 339 do_select_test 1.$i.asc { |
| 321 SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term | 340 SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term |
| 322 } [simple_token_matchinfo $term] | 341 ORDER BY docid ASC |
| 342 } [simple_token_matchinfo $term 0] |
| 343 do_select_test 1.$i.desc { |
| 344 SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term |
| 345 ORDER BY docid DESC |
| 346 } [simple_token_matchinfo $term 1] |
| 323 } | 347 } |
| 324 | 348 |
| 325 # This time, use the first two characters of each term as a term prefix | 349 # This time, use the first two characters of each term as a term prefix |
| 326 # to query for. Test that querying the Tcl array produces the same results | 350 # to query for. Test that querying the Tcl array produces the same results |
| 327 # as querying the FTS3 table for the prefix. | 351 # as querying the FTS3 table for the prefix. |
| 328 # | 352 # |
| 329 for {set i 0} {$i < $nRep} {incr i} { | 353 for {set i 0} {$i < $nRep} {incr i} { |
| 330 set prefix [string range [random_term] 0 end-1] | 354 set prefix [string range [random_term] 0 end-1] |
| 331 set match "${prefix}*" | 355 set match "${prefix}*" |
| 332 do_select_test fts3rnd-1.$nodesize.$iTest.2.$i { | 356 do_orderbydocid_test 2.$i { |
| 333 SELECT docid FROM t1 WHERE t1 MATCH $match | 357 SELECT docid FROM t1 WHERE t1 MATCH $match |
| 334 } [simple_phrase $match] | 358 } [simple_phrase $match] |
| 335 } | 359 } |
| 336 | 360 |
| 337 # Similar to the above, except for phrase queries. | 361 # Similar to the above, except for phrase queries. |
| 338 # | 362 # |
| 339 for {set i 0} {$i < $nRep} {incr i} { | 363 for {set i 0} {$i < $nRep} {incr i} { |
| 340 set term [list [random_term] [random_term]] | 364 set term [list [random_term] [random_term]] |
| 341 set match "\"$term\"" | 365 set match "\"$term\"" |
| 342 do_select_test fts3rnd-1.$nodesize.$iTest.3.$i { | 366 do_orderbydocid_test 3.$i { |
| 343 SELECT docid FROM t1 WHERE t1 MATCH $match | 367 SELECT docid FROM t1 WHERE t1 MATCH $match |
| 344 } [simple_phrase $term] | 368 } [simple_phrase $term] |
| 345 } | 369 } |
| 346 | 370 |
| 347 # Three word phrases. | 371 # Three word phrases. |
| 348 # | 372 # |
| 349 for {set i 0} {$i < $nRep} {incr i} { | 373 for {set i 0} {$i < $nRep} {incr i} { |
| 350 set term [list [random_term] [random_term] [random_term]] | 374 set term [list [random_term] [random_term] [random_term]] |
| 351 set match "\"$term\"" | 375 set match "\"$term\"" |
| 352 do_select_test fts3rnd-1.$nodesize.$iTest.4.$i { | 376 do_orderbydocid_test 4.$i { |
| 353 SELECT docid FROM t1 WHERE t1 MATCH $match | 377 SELECT docid FROM t1 WHERE t1 MATCH $match |
| 354 } [simple_phrase $term] | 378 } [simple_phrase $term] |
| 355 } | 379 } |
| 356 | 380 |
| 357 # Three word phrases made up of term-prefixes. | 381 # Three word phrases made up of term-prefixes. |
| 358 # | 382 # |
| 359 for {set i 0} {$i < $nRep} {incr i} { | 383 for {set i 0} {$i < $nRep} {incr i} { |
| 360 set query "[string range [random_term] 0 end-1]* " | 384 set query "[string range [random_term] 0 end-1]* " |
| 361 append query "[string range [random_term] 0 end-1]* " | 385 append query "[string range [random_term] 0 end-1]* " |
| 362 append query "[string range [random_term] 0 end-1]*" | 386 append query "[string range [random_term] 0 end-1]*" |
| 363 | 387 |
| 364 set match "\"$query\"" | 388 set match "\"$query\"" |
| 365 do_select_test fts3rnd-1.$nodesize.$iTest.5.$i { | 389 do_orderbydocid_test 5.$i { |
| 366 SELECT docid FROM t1 WHERE t1 MATCH $match | 390 SELECT docid FROM t1 WHERE t1 MATCH $match |
| 367 } [simple_phrase $query] | 391 } [simple_phrase $query] |
| 368 } | 392 } |
| 369 | 393 |
| 370 # A NEAR query with terms as the arguments. | 394 # A NEAR query with terms as the arguments: |
| 395 # |
| 396 # ... MATCH '$term1 NEAR $term2' ... |
| 371 # | 397 # |
| 372 for {set i 0} {$i < $nRep} {incr i} { | 398 for {set i 0} {$i < $nRep} {incr i} { |
| 373 set terms [list [random_term] [random_term]] | 399 set terms [list [random_term] [random_term]] |
| 374 set match [join $terms " NEAR "] | 400 set match [join $terms " NEAR "] |
| 375 do_select_test fts3rnd-1.$nodesize.$iTest.6.$i { | 401 do_orderbydocid_test 6.$i { |
| 376 SELECT docid FROM t1 WHERE t1 MATCH $match | 402 SELECT docid FROM t1 WHERE t1 MATCH $match |
| 377 } [simple_near $terms 10] | 403 } [simple_near $terms 10] |
| 378 } | 404 } |
| 379 | 405 |
| 380 # A 3-way NEAR query with terms as the arguments. | 406 # A 3-way NEAR query with terms as the arguments. |
| 381 # | 407 # |
| 382 for {set i 0} {$i < $nRep} {incr i} { | 408 for {set i 0} {$i < $nRep} {incr i} { |
| 383 set terms [list [random_term] [random_term] [random_term]] | 409 set terms [list [random_term] [random_term] [random_term]] |
| 384 set nNear 11 | 410 set nNear 11 |
| 385 set match [join $terms " NEAR/$nNear "] | 411 set match [join $terms " NEAR/$nNear "] |
| 386 do_select_test fts3rnd-1.$nodesize.$iTest.7.$i { | 412 do_orderbydocid_test 7.$i { |
| 387 SELECT docid FROM t1 WHERE t1 MATCH $match | 413 SELECT docid FROM t1 WHERE t1 MATCH $match |
| 388 } [simple_near $terms $nNear] | 414 } [simple_near $terms $nNear] |
| 389 } | 415 } |
| 390 | 416 |
| 391 # Set operations on simple term queries. | 417 # Set operations on simple term queries. |
| 392 # | 418 # |
| 393 foreach {tn op proc} { | 419 foreach {tn op proc} { |
| 394 8 OR setop_or | 420 8 OR setop_or |
| 395 9 NOT setop_not | 421 9 NOT setop_not |
| 396 10 AND setop_and | 422 10 AND setop_and |
| 397 } { | 423 } { |
| 398 for {set i 0} {$i < $nRep} {incr i} { | 424 for {set i 0} {$i < $nRep} {incr i} { |
| 399 set term1 [random_term] | 425 set term1 [random_term] |
| 400 set term2 [random_term] | 426 set term2 [random_term] |
| 401 set match "$term1 $op $term2" | 427 set match "$term1 $op $term2" |
| 402 do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { | 428 do_orderbydocid_test $tn.$i { |
| 403 SELECT docid FROM t1 WHERE t1 MATCH $match | 429 SELECT docid FROM t1 WHERE t1 MATCH $match |
| 404 } [$proc [simple_phrase $term1] [simple_phrase $term2]] | 430 } [$proc [simple_phrase $term1] [simple_phrase $term2]] |
| 405 } | 431 } |
| 406 } | 432 } |
| 407 | 433 |
| 408 # Set operations on NEAR queries. | 434 # Set operations on NEAR queries. |
| 409 # | 435 # |
| 410 foreach {tn op proc} { | 436 foreach {tn op proc} { |
| 411 8 OR setop_or | 437 11 OR setop_or |
| 412 9 NOT setop_not | 438 12 NOT setop_not |
| 413 10 AND setop_and | 439 13 AND setop_and |
| 414 } { | 440 } { |
| 415 for {set i 0} {$i < $nRep} {incr i} { | 441 for {set i 0} {$i < $nRep} {incr i} { |
| 416 set term1 [random_term] | 442 set term1 [random_term] |
| 417 set term2 [random_term] | 443 set term2 [random_term] |
| 418 set term3 [random_term] | 444 set term3 [random_term] |
| 419 set term4 [random_term] | 445 set term4 [random_term] |
| 420 set match "$term1 NEAR $term2 $op $term3 NEAR $term4" | 446 set match "$term1 NEAR $term2 $op $term3 NEAR $term4" |
| 421 do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { | 447 do_orderbydocid_test $tn.$i { |
| 422 SELECT docid FROM t1 WHERE t1 MATCH $match | 448 SELECT docid FROM t1 WHERE t1 MATCH $match |
| 423 } [$proc \ | 449 } [$proc \ |
| 424 [simple_near [list $term1 $term2] 10] \ | 450 [simple_near [list $term1 $term2] 10] \ |
| 425 [simple_near [list $term3 $term4] 10] | 451 [simple_near [list $term3 $term4] 10] |
| 426 ] | 452 ] |
| 427 } | 453 } |
| 428 } | 454 } |
| 429 | 455 |
| 430 catchsql COMMIT | 456 catchsql COMMIT |
| 431 } | 457 } |
| 432 } | 458 } |
| 433 | 459 |
| 434 finish_test | 460 finish_test |
| OLD | NEW |