OLD | NEW |
1 # 2009 December 03 | 1 # 2009 December 03 |
2 # | 2 # |
3 # May you do good and not evil. | 3 # May you do good and not evil. |
4 # May you find forgiveness for yourself and forgive others. | 4 # May you find forgiveness for yourself and forgive others. |
5 # May you share freely, never taking more than you give. | 5 # May you share freely, never taking more than you give. |
6 # | 6 # |
7 #*********************************************************************** | 7 #*********************************************************************** |
8 # | 8 # |
9 # Brute force (random data) tests for FTS3. | 9 # Brute force (random data) tests for FTS3. |
10 # | 10 # |
(...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
155 if {[regexp $reg " $col "]} { lappend ret $key ; break } | 155 if {[regexp $reg " $col "]} { lappend ret $key ; break } |
156 } | 156 } |
157 } | 157 } |
158 | 158 |
159 #lsort -uniq -integer $ret | 159 #lsort -uniq -integer $ret |
160 set ret | 160 set ret |
161 } | 161 } |
162 | 162 |
163 # This [proc] is used to test the FTS3 matchinfo() function. | 163 # This [proc] is used to test the FTS3 matchinfo() function. |
164 # | 164 # |
165 proc simple_token_matchinfo {zToken} { | 165 proc simple_token_matchinfo {zToken bDesc} { |
166 | 166 |
167 set nDoc(0) 0 | 167 set nDoc(0) 0 |
168 set nDoc(1) 0 | 168 set nDoc(1) 0 |
169 set nDoc(2) 0 | 169 set nDoc(2) 0 |
170 set nHit(0) 0 | 170 set nHit(0) 0 |
171 set nHit(1) 0 | 171 set nHit(1) 0 |
172 set nHit(2) 0 | 172 set nHit(2) 0 |
173 | 173 |
| 174 set dir -inc |
| 175 if {$bDesc} { set dir -dec } |
174 | 176 |
175 foreach key [array names ::t1] { | 177 foreach key [array names ::t1] { |
176 set value $::t1($key) | 178 set value $::t1($key) |
177 set a($key) [list] | 179 set a($key) [list] |
178 foreach i {0 1 2} col $value { | 180 foreach i {0 1 2} col $value { |
179 set hit [llength [lsearch -all $col $zToken]] | 181 set hit [llength [lsearch -all $col $zToken]] |
180 lappend a($key) $hit | 182 lappend a($key) $hit |
181 incr nHit($i) $hit | 183 incr nHit($i) $hit |
182 if {$hit>0} { incr nDoc($i) } | 184 if {$hit>0} { incr nDoc($i) } |
183 } | 185 } |
184 } | 186 } |
185 | 187 |
186 set ret [list] | 188 set ret [list] |
187 foreach docid [lsort -integer [array names a]] { | 189 foreach docid [lsort -integer $dir [array names a]] { |
188 if { [lindex [lsort -integer $a($docid)] end] } { | 190 if { [lindex [lsort -integer $a($docid)] end] } { |
189 set matchinfo [list 1 3] | 191 set matchinfo [list 1 3] |
190 foreach i {0 1 2} hit $a($docid) { | 192 foreach i {0 1 2} hit $a($docid) { |
191 lappend matchinfo $hit $nHit($i) $nDoc($i) | 193 lappend matchinfo $hit $nHit($i) $nDoc($i) |
192 } | 194 } |
193 lappend ret $docid $matchinfo | 195 lappend ret $docid $matchinfo |
194 } | 196 } |
195 } | 197 } |
196 | 198 |
197 set ret | 199 set ret |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
255 return $ret | 257 return $ret |
256 } | 258 } |
257 | 259 |
258 proc mit {blob} { | 260 proc mit {blob} { |
259 set scan(littleEndian) i* | 261 set scan(littleEndian) i* |
260 set scan(bigEndian) I* | 262 set scan(bigEndian) I* |
261 binary scan $blob $scan($::tcl_platform(byteOrder)) r | 263 binary scan $blob $scan($::tcl_platform(byteOrder)) r |
262 return $r | 264 return $r |
263 } | 265 } |
264 db func mit mit | 266 db func mit mit |
265 | |
266 set sqlite_fts3_enable_parentheses 1 | 267 set sqlite_fts3_enable_parentheses 1 |
267 | 268 |
268 foreach nodesize {50 500 1000 2000} { | 269 proc do_orderbydocid_test {tn sql res} { |
| 270 uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res] |
| 271 uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \ |
| 272 [lsort -int -dec $res] |
| 273 ] |
| 274 } |
| 275 |
| 276 set NUM_TRIALS 100 |
| 277 |
| 278 foreach {nodesize order} { |
| 279 50 DESC |
| 280 50 ASC |
| 281 500 ASC |
| 282 1000 DESC |
| 283 2000 ASC |
| 284 } { |
269 catch { array unset ::t1 } | 285 catch { array unset ::t1 } |
| 286 set testname "$nodesize/$order" |
270 | 287 |
271 # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. | 288 # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. |
272 # | 289 # |
273 db transaction { | 290 db transaction { |
274 catchsql { DROP TABLE t1 } | 291 catchsql { DROP TABLE t1 } |
275 execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)" | 292 execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)" |
276 execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')" | 293 execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')" |
277 for {set i 0} {$i < 100} {incr i} { insert_row $i } | 294 for {set i 0} {$i < 100} {incr i} { insert_row $i } |
278 } | 295 } |
279 | 296 |
280 for {set iTest 1} {$iTest <= 100} {incr iTest} { | 297 for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} { |
281 catchsql COMMIT | 298 catchsql COMMIT |
282 | 299 |
283 set DO_MALLOC_TEST 0 | 300 set DO_MALLOC_TEST 0 |
284 set nRep 10 | 301 set nRep 10 |
285 if {$iTest==100 && $nodesize==50} { | 302 if {$iTest==100 && $nodesize==50} { |
286 set DO_MALLOC_TEST 1 | 303 set DO_MALLOC_TEST 1 |
287 set nRep 2 | 304 set nRep 2 |
288 } | 305 } |
| 306 |
| 307 set ::testprefix fts3rnd-1.$testname.$iTest |
289 | 308 |
290 # Delete one row, update one row and insert one row. | 309 # Delete one row, update one row and insert one row. |
291 # | 310 # |
292 set rows [array names ::t1] | 311 set rows [array names ::t1] |
293 set nRow [llength $rows] | 312 set nRow [llength $rows] |
294 set iUpdate [lindex $rows [expr {int(rand()*$nRow)}]] | 313 set iUpdate [lindex $rows [expr {int(rand()*$nRow)}]] |
295 set iDelete $iUpdate | 314 set iDelete $iUpdate |
296 while {$iDelete == $iUpdate} { | 315 while {$iDelete == $iUpdate} { |
297 set iDelete [lindex $rows [expr {int(rand()*$nRow)}]] | 316 set iDelete [lindex $rows [expr {int(rand()*$nRow)}]] |
298 } | 317 } |
299 set iInsert $iUpdate | 318 set iInsert $iUpdate |
300 while {[info exists ::t1($iInsert)]} { | 319 while {[info exists ::t1($iInsert)]} { |
301 set iInsert [expr {int(rand()*1000000)}] | 320 set iInsert [expr {int(rand()*1000000)}] |
302 } | 321 } |
303 execsql BEGIN | 322 execsql BEGIN |
304 insert_row $iInsert | 323 insert_row $iInsert |
305 update_row $iUpdate | 324 update_row $iUpdate |
306 delete_row $iDelete | 325 delete_row $iDelete |
307 if {0==($iTest%2)} { execsql COMMIT } | 326 if {0==($iTest%2)} { execsql COMMIT } |
308 | 327 |
309 if {0==($iTest%2)} { | 328 if {0==($iTest%2)} { |
310 do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok | 329 #do_test 0 { fts3_integrity_check t1 } ok |
311 } | 330 } |
312 | 331 |
313 # Pick 10 terms from the vocabulary. Check that the results of querying | 332 # Pick 10 terms from the vocabulary. Check that the results of querying |
314 # the database for the set of documents containing each of these terms | 333 # the database for the set of documents containing each of these terms |
315 # is the same as the result obtained by scanning the contents of the Tcl | 334 # is the same as the result obtained by scanning the contents of the Tcl |
316 # array for each term. | 335 # array for each term. |
317 # | 336 # |
318 for {set i 0} {$i < 10} {incr i} { | 337 for {set i 0} {$i < 10} {incr i} { |
319 set term [random_term] | 338 set term [random_term] |
320 do_select_test fts3rnd-1.$nodesize.$iTest.1.$i { | 339 do_select_test 1.$i.asc { |
321 SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term | 340 SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term |
322 } [simple_token_matchinfo $term] | 341 ORDER BY docid ASC |
| 342 } [simple_token_matchinfo $term 0] |
| 343 do_select_test 1.$i.desc { |
| 344 SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term |
| 345 ORDER BY docid DESC |
| 346 } [simple_token_matchinfo $term 1] |
323 } | 347 } |
324 | 348 |
325 # This time, use the first two characters of each term as a term prefix | 349 # This time, use the first two characters of each term as a term prefix |
326 # to query for. Test that querying the Tcl array produces the same results | 350 # to query for. Test that querying the Tcl array produces the same results |
327 # as querying the FTS3 table for the prefix. | 351 # as querying the FTS3 table for the prefix. |
328 # | 352 # |
329 for {set i 0} {$i < $nRep} {incr i} { | 353 for {set i 0} {$i < $nRep} {incr i} { |
330 set prefix [string range [random_term] 0 end-1] | 354 set prefix [string range [random_term] 0 end-1] |
331 set match "${prefix}*" | 355 set match "${prefix}*" |
332 do_select_test fts3rnd-1.$nodesize.$iTest.2.$i { | 356 do_orderbydocid_test 2.$i { |
333 SELECT docid FROM t1 WHERE t1 MATCH $match | 357 SELECT docid FROM t1 WHERE t1 MATCH $match |
334 } [simple_phrase $match] | 358 } [simple_phrase $match] |
335 } | 359 } |
336 | 360 |
337 # Similar to the above, except for phrase queries. | 361 # Similar to the above, except for phrase queries. |
338 # | 362 # |
339 for {set i 0} {$i < $nRep} {incr i} { | 363 for {set i 0} {$i < $nRep} {incr i} { |
340 set term [list [random_term] [random_term]] | 364 set term [list [random_term] [random_term]] |
341 set match "\"$term\"" | 365 set match "\"$term\"" |
342 do_select_test fts3rnd-1.$nodesize.$iTest.3.$i { | 366 do_orderbydocid_test 3.$i { |
343 SELECT docid FROM t1 WHERE t1 MATCH $match | 367 SELECT docid FROM t1 WHERE t1 MATCH $match |
344 } [simple_phrase $term] | 368 } [simple_phrase $term] |
345 } | 369 } |
346 | 370 |
347 # Three word phrases. | 371 # Three word phrases. |
348 # | 372 # |
349 for {set i 0} {$i < $nRep} {incr i} { | 373 for {set i 0} {$i < $nRep} {incr i} { |
350 set term [list [random_term] [random_term] [random_term]] | 374 set term [list [random_term] [random_term] [random_term]] |
351 set match "\"$term\"" | 375 set match "\"$term\"" |
352 do_select_test fts3rnd-1.$nodesize.$iTest.4.$i { | 376 do_orderbydocid_test 4.$i { |
353 SELECT docid FROM t1 WHERE t1 MATCH $match | 377 SELECT docid FROM t1 WHERE t1 MATCH $match |
354 } [simple_phrase $term] | 378 } [simple_phrase $term] |
355 } | 379 } |
356 | 380 |
357 # Three word phrases made up of term-prefixes. | 381 # Three word phrases made up of term-prefixes. |
358 # | 382 # |
359 for {set i 0} {$i < $nRep} {incr i} { | 383 for {set i 0} {$i < $nRep} {incr i} { |
360 set query "[string range [random_term] 0 end-1]* " | 384 set query "[string range [random_term] 0 end-1]* " |
361 append query "[string range [random_term] 0 end-1]* " | 385 append query "[string range [random_term] 0 end-1]* " |
362 append query "[string range [random_term] 0 end-1]*" | 386 append query "[string range [random_term] 0 end-1]*" |
363 | 387 |
364 set match "\"$query\"" | 388 set match "\"$query\"" |
365 do_select_test fts3rnd-1.$nodesize.$iTest.5.$i { | 389 do_orderbydocid_test 5.$i { |
366 SELECT docid FROM t1 WHERE t1 MATCH $match | 390 SELECT docid FROM t1 WHERE t1 MATCH $match |
367 } [simple_phrase $query] | 391 } [simple_phrase $query] |
368 } | 392 } |
369 | 393 |
370 # A NEAR query with terms as the arguments. | 394 # A NEAR query with terms as the arguments: |
| 395 # |
| 396 # ... MATCH '$term1 NEAR $term2' ... |
371 # | 397 # |
372 for {set i 0} {$i < $nRep} {incr i} { | 398 for {set i 0} {$i < $nRep} {incr i} { |
373 set terms [list [random_term] [random_term]] | 399 set terms [list [random_term] [random_term]] |
374 set match [join $terms " NEAR "] | 400 set match [join $terms " NEAR "] |
375 do_select_test fts3rnd-1.$nodesize.$iTest.6.$i { | 401 do_orderbydocid_test 6.$i { |
376 SELECT docid FROM t1 WHERE t1 MATCH $match | 402 SELECT docid FROM t1 WHERE t1 MATCH $match |
377 } [simple_near $terms 10] | 403 } [simple_near $terms 10] |
378 } | 404 } |
379 | 405 |
380 # A 3-way NEAR query with terms as the arguments. | 406 # A 3-way NEAR query with terms as the arguments. |
381 # | 407 # |
382 for {set i 0} {$i < $nRep} {incr i} { | 408 for {set i 0} {$i < $nRep} {incr i} { |
383 set terms [list [random_term] [random_term] [random_term]] | 409 set terms [list [random_term] [random_term] [random_term]] |
384 set nNear 11 | 410 set nNear 11 |
385 set match [join $terms " NEAR/$nNear "] | 411 set match [join $terms " NEAR/$nNear "] |
386 do_select_test fts3rnd-1.$nodesize.$iTest.7.$i { | 412 do_orderbydocid_test 7.$i { |
387 SELECT docid FROM t1 WHERE t1 MATCH $match | 413 SELECT docid FROM t1 WHERE t1 MATCH $match |
388 } [simple_near $terms $nNear] | 414 } [simple_near $terms $nNear] |
389 } | 415 } |
390 | 416 |
391 # Set operations on simple term queries. | 417 # Set operations on simple term queries. |
392 # | 418 # |
393 foreach {tn op proc} { | 419 foreach {tn op proc} { |
394 8 OR setop_or | 420 8 OR setop_or |
395 9 NOT setop_not | 421 9 NOT setop_not |
396 10 AND setop_and | 422 10 AND setop_and |
397 } { | 423 } { |
398 for {set i 0} {$i < $nRep} {incr i} { | 424 for {set i 0} {$i < $nRep} {incr i} { |
399 set term1 [random_term] | 425 set term1 [random_term] |
400 set term2 [random_term] | 426 set term2 [random_term] |
401 set match "$term1 $op $term2" | 427 set match "$term1 $op $term2" |
402 do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { | 428 do_orderbydocid_test $tn.$i { |
403 SELECT docid FROM t1 WHERE t1 MATCH $match | 429 SELECT docid FROM t1 WHERE t1 MATCH $match |
404 } [$proc [simple_phrase $term1] [simple_phrase $term2]] | 430 } [$proc [simple_phrase $term1] [simple_phrase $term2]] |
405 } | 431 } |
406 } | 432 } |
407 | 433 |
408 # Set operations on NEAR queries. | 434 # Set operations on NEAR queries. |
409 # | 435 # |
410 foreach {tn op proc} { | 436 foreach {tn op proc} { |
411 8 OR setop_or | 437 11 OR setop_or |
412 9 NOT setop_not | 438 12 NOT setop_not |
413 10 AND setop_and | 439 13 AND setop_and |
414 } { | 440 } { |
415 for {set i 0} {$i < $nRep} {incr i} { | 441 for {set i 0} {$i < $nRep} {incr i} { |
416 set term1 [random_term] | 442 set term1 [random_term] |
417 set term2 [random_term] | 443 set term2 [random_term] |
418 set term3 [random_term] | 444 set term3 [random_term] |
419 set term4 [random_term] | 445 set term4 [random_term] |
420 set match "$term1 NEAR $term2 $op $term3 NEAR $term4" | 446 set match "$term1 NEAR $term2 $op $term3 NEAR $term4" |
421 do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { | 447 do_orderbydocid_test $tn.$i { |
422 SELECT docid FROM t1 WHERE t1 MATCH $match | 448 SELECT docid FROM t1 WHERE t1 MATCH $match |
423 } [$proc \ | 449 } [$proc \ |
424 [simple_near [list $term1 $term2] 10] \ | 450 [simple_near [list $term1 $term2] 10] \ |
425 [simple_near [list $term3 $term4] 10] | 451 [simple_near [list $term3 $term4] 10] |
426 ] | 452 ] |
427 } | 453 } |
428 } | 454 } |
429 | 455 |
430 catchsql COMMIT | 456 catchsql COMMIT |
431 } | 457 } |
432 } | 458 } |
433 | 459 |
434 finish_test | 460 finish_test |
OLD | NEW |