Index: third_party/sqlite/src/test/fts3snippet.test |
diff --git a/third_party/sqlite/src/test/fts3snippet.test b/third_party/sqlite/src/test/fts3snippet.test |
new file mode 100644 |
index 0000000000000000000000000000000000000000..359a87c6f318005c3fbc8d0ba2211a5892b5339a |
--- /dev/null |
+++ b/third_party/sqlite/src/test/fts3snippet.test |
@@ -0,0 +1,463 @@ |
+# 2010 January 07 |
+# |
+# The author disclaims copyright to this source code. In place of |
+# a legal notice, here is a blessing: |
+# |
+# May you do good and not evil. |
+# May you find forgiveness for yourself and forgive others. |
+# May you share freely, never taking more than you give. |
+# |
+#************************************************************************* |
+# |
+# The tests in this file test the FTS3 auxillary functions offsets(), |
+# snippet() and matchinfo() work. At time of writing, running this file |
+# provides full coverage of fts3_snippet.c. |
+# |
+ |
+set testdir [file dirname $argv0] |
+source $testdir/tester.tcl |
+ |
+# If SQLITE_ENABLE_FTS3 is not defined, omit this file. |
+ifcapable !fts3 { finish_test ; return } |
+source $testdir/fts3_common.tcl |
+ |
+set sqlite_fts3_enable_parentheses 1 |
+set DO_MALLOC_TEST 0 |
+ |
+# Transform the list $L to its "normal" form. So that it can be compared to |
+# another list with the same set of elements using [string compare]. |
+# |
+proc normalize {L} { |
+ set ret [list] |
+ foreach l $L {lappend ret $l} |
+ return $ret |
+} |
+ |
+proc do_offsets_test {name expr args} { |
+ set result [list] |
+ foreach a $args { |
+ lappend result [normalize $a] |
+ } |
+ do_select_test $name { |
+ SELECT offsets(ft) FROM ft WHERE ft MATCH $expr |
+ } $result |
+} |
+ |
+# Document text used by a few tests. Contains the English names of all |
+# integers between 1 and 300. |
+# |
+set numbers [normalize { |
+ one two three four five six seven eight nine ten eleven twelve thirteen |
+ fourteen fifteen sixteen seventeen eighteen nineteen twenty twentyone |
+ twentytwo twentythree twentyfour twentyfive twentysix twentyseven |
+ twentyeight twentynine thirty thirtyone thirtytwo thirtythree thirtyfour |
+ thirtyfive thirtysix thirtyseven thirtyeight thirtynine forty fortyone |
+ fortytwo fortythree fortyfour fortyfive fortysix fortyseven fortyeight |
+ fortynine fifty fiftyone fiftytwo fiftythree fiftyfour fiftyfive fiftysix |
+ fiftyseven fiftyeight fiftynine sixty sixtyone sixtytwo sixtythree sixtyfour |
+ sixtyfive sixtysix sixtyseven sixtyeight sixtynine seventy seventyone |
+ seventytwo seventythree seventyfour seventyfive seventysix seventyseven |
+ seventyeight seventynine eighty eightyone eightytwo eightythree eightyfour |
+ eightyfive eightysix eightyseven eightyeight eightynine ninety ninetyone |
+ ninetytwo ninetythree ninetyfour ninetyfive ninetysix ninetyseven |
+ ninetyeight ninetynine onehundred onehundredone onehundredtwo |
+ onehundredthree onehundredfour onehundredfive onehundredsix onehundredseven |
+ onehundredeight onehundrednine onehundredten onehundredeleven |
+ onehundredtwelve onehundredthirteen onehundredfourteen onehundredfifteen |
+ onehundredsixteen onehundredseventeen onehundredeighteen onehundrednineteen |
+ onehundredtwenty onehundredtwentyone onehundredtwentytwo |
+ onehundredtwentythree onehundredtwentyfour onehundredtwentyfive |
+ onehundredtwentysix onehundredtwentyseven onehundredtwentyeight |
+ onehundredtwentynine onehundredthirty onehundredthirtyone |
+ onehundredthirtytwo onehundredthirtythree onehundredthirtyfour |
+ onehundredthirtyfive onehundredthirtysix onehundredthirtyseven |
+ onehundredthirtyeight onehundredthirtynine onehundredforty |
+ onehundredfortyone onehundredfortytwo onehundredfortythree |
+ onehundredfortyfour onehundredfortyfive onehundredfortysix |
+ onehundredfortyseven onehundredfortyeight onehundredfortynine |
+ onehundredfifty onehundredfiftyone onehundredfiftytwo onehundredfiftythree |
+ onehundredfiftyfour onehundredfiftyfive onehundredfiftysix |
+ onehundredfiftyseven onehundredfiftyeight onehundredfiftynine |
+ onehundredsixty onehundredsixtyone onehundredsixtytwo onehundredsixtythree |
+ onehundredsixtyfour onehundredsixtyfive onehundredsixtysix |
+ onehundredsixtyseven onehundredsixtyeight onehundredsixtynine |
+ onehundredseventy onehundredseventyone onehundredseventytwo |
+ onehundredseventythree onehundredseventyfour onehundredseventyfive |
+ onehundredseventysix onehundredseventyseven onehundredseventyeight |
+ onehundredseventynine onehundredeighty onehundredeightyone |
+ onehundredeightytwo onehundredeightythree onehundredeightyfour |
+ onehundredeightyfive onehundredeightysix onehundredeightyseven |
+ onehundredeightyeight onehundredeightynine onehundredninety |
+ onehundredninetyone onehundredninetytwo onehundredninetythree |
+ onehundredninetyfour onehundredninetyfive onehundredninetysix |
+ onehundredninetyseven onehundredninetyeight onehundredninetynine twohundred |
+ twohundredone twohundredtwo twohundredthree twohundredfour twohundredfive |
+ twohundredsix twohundredseven twohundredeight twohundrednine twohundredten |
+ twohundredeleven twohundredtwelve twohundredthirteen twohundredfourteen |
+ twohundredfifteen twohundredsixteen twohundredseventeen twohundredeighteen |
+ twohundrednineteen twohundredtwenty twohundredtwentyone twohundredtwentytwo |
+ twohundredtwentythree twohundredtwentyfour twohundredtwentyfive |
+ twohundredtwentysix twohundredtwentyseven twohundredtwentyeight |
+ twohundredtwentynine twohundredthirty twohundredthirtyone |
+ twohundredthirtytwo twohundredthirtythree twohundredthirtyfour |
+ twohundredthirtyfive twohundredthirtysix twohundredthirtyseven |
+ twohundredthirtyeight twohundredthirtynine twohundredforty |
+ twohundredfortyone twohundredfortytwo twohundredfortythree |
+ twohundredfortyfour twohundredfortyfive twohundredfortysix |
+ twohundredfortyseven twohundredfortyeight twohundredfortynine |
+ twohundredfifty twohundredfiftyone twohundredfiftytwo twohundredfiftythree |
+ twohundredfiftyfour twohundredfiftyfive twohundredfiftysix |
+ twohundredfiftyseven twohundredfiftyeight twohundredfiftynine |
+ twohundredsixty twohundredsixtyone twohundredsixtytwo twohundredsixtythree |
+ twohundredsixtyfour twohundredsixtyfive twohundredsixtysix |
+ twohundredsixtyseven twohundredsixtyeight twohundredsixtynine |
+ twohundredseventy twohundredseventyone twohundredseventytwo |
+ twohundredseventythree twohundredseventyfour twohundredseventyfive |
+ twohundredseventysix twohundredseventyseven twohundredseventyeight |
+ twohundredseventynine twohundredeighty twohundredeightyone |
+ twohundredeightytwo twohundredeightythree twohundredeightyfour |
+ twohundredeightyfive twohundredeightysix twohundredeightyseven |
+ twohundredeightyeight twohundredeightynine twohundredninety |
+ twohundredninetyone twohundredninetytwo twohundredninetythree |
+ twohundredninetyfour twohundredninetyfive twohundredninetysix |
+ twohundredninetyseven twohundredninetyeight twohundredninetynine |
+ threehundred |
+}] |
+ |
+foreach {DO_MALLOC_TEST enc} { |
+ 0 utf8 |
+ 1 utf8 |
+ 1 utf16 |
+} { |
+ |
+ db close |
+ file delete -force test.db |
+ sqlite3 db test.db |
+ sqlite3_db_config_lookaside db 0 0 0 |
+ db eval "PRAGMA encoding = \"$enc\"" |
+ |
+ # Set variable $T to the test name prefix for this iteration of the loop. |
+ # |
+ set T "fts3snippet-$enc" |
+ |
+ ########################################################################## |
+ # Test the offset function. |
+ # |
+ do_test $T.1.1 { |
+ execsql { |
+ CREATE VIRTUAL TABLE ft USING fts3; |
+ INSERT INTO ft VALUES('xxx xxx xxx xxx'); |
+ } |
+ } {} |
+ do_offsets_test $T.1.2 {xxx} {0 0 0 3 0 0 4 3 0 0 8 3 0 0 12 3} |
+ do_offsets_test $T.1.3 {"xxx xxx"} { |
+ 0 0 0 3 0 0 4 3 0 1 4 3 0 0 8 3 |
+ 0 1 8 3 0 1 12 3 |
+ } |
+ do_offsets_test $T.1.4 {"xxx xxx" xxx} { |
+ 0 0 0 3 0 2 0 3 0 0 4 3 0 1 4 3 |
+ 0 2 4 3 0 0 8 3 0 1 8 3 0 2 8 3 |
+ 0 1 12 3 0 2 12 3 |
+ } |
+ do_offsets_test $T.1.5 {xxx "xxx xxx"} { |
+ 0 0 0 3 0 1 0 3 0 0 4 3 0 1 4 3 |
+ 0 2 4 3 0 0 8 3 0 1 8 3 0 2 8 3 |
+ 0 0 12 3 0 2 12 3 |
+ } |
+ |
+ do_test $T.2.1 { |
+ set v1 [lrange $numbers 0 99] |
+ execsql { |
+ DROP TABLE IF EXISTS ft; |
+ CREATE VIRTUAL TABLE ft USING fts3(a, b); |
+ INSERT INTO ft VALUES($v1, $numbers); |
+ INSERT INTO ft VALUES($v1, NULL); |
+ } |
+ } {} |
+ |
+ set off [string first "twohundred " $numbers] |
+ do_offsets_test $T.2.1 {twohundred} [list 1 0 $off 10] |
+ |
+ set off [string first "onehundred " $numbers] |
+ do_offsets_test $T.2.2 {onehundred} \ |
+ [list 0 0 $off 10 1 0 $off 10] [list 0 0 $off 10] |
+ |
+ # Test a corruption case: |
+ execsql { UPDATE ft_content SET c1b = 'hello world' WHERE c1b = $numbers } |
+ do_error_test $T.2.3 { |
+ SELECT offsets(ft) FROM ft WHERE ft MATCH 'onehundred' |
+ } {database disk image is malformed} |
+ |
+ ########################################################################## |
+ # Test the snippet function. |
+ # |
+ proc do_snippet_test {name expr iCol nTok args} { |
+ set res [list] |
+ foreach a $args { lappend res [string trim $a] } |
+ do_select_test $name { |
+ SELECT snippet(ft,'{','}','...',$iCol,$nTok) FROM ft WHERE ft MATCH $expr |
+ } $res |
+ } |
+ do_test $T.3.1 { |
+ execsql { |
+ DROP TABLE IF EXISTS ft; |
+ CREATE VIRTUAL TABLE ft USING fts3; |
+ INSERT INTO ft VALUES('one two three four five six seven eight nine ten'); |
+ } |
+ } {} |
+ do_snippet_test $T.3.2 one 0 5 "{one} two three four five..." |
+ do_snippet_test $T.3.3 two 0 5 "one {two} three four five..." |
+ do_snippet_test $T.3.4 three 0 5 "one two {three} four five..." |
+ do_snippet_test $T.3.5 four 0 5 "...two three {four} five six..." |
+ do_snippet_test $T.3.6 five 0 5 "...three four {five} six seven..." |
+ do_snippet_test $T.3.7 six 0 5 "...four five {six} seven eight..." |
+ do_snippet_test $T.3.8 seven 0 5 "...five six {seven} eight nine..." |
+ do_snippet_test $T.3.9 eight 0 5 "...six seven {eight} nine ten" |
+ do_snippet_test $T.3.10 nine 0 5 "...six seven eight {nine} ten" |
+ do_snippet_test $T.3.11 ten 0 5 "...six seven eight nine {ten}" |
+ |
+ do_test $T.4.1 { |
+ execsql { |
+ INSERT INTO ft VALUES( |
+ 'one two three four five ' |
+ || 'six seven eight nine ten ' |
+ || 'eleven twelve thirteen fourteen fifteen ' |
+ || 'sixteen seventeen eighteen nineteen twenty ' |
+ || 'one two three four five ' |
+ || 'six seven eight nine ten ' |
+ || 'eleven twelve thirteen fourteen fifteen ' |
+ || 'sixteen seventeen eighteen nineteen twenty' |
+ ); |
+ } |
+ } {} |
+ |
+ do_snippet_test $T.4.2 {one nine} 0 5 { |
+ {one} two three...eight {nine} ten |
+ } { |
+ {one} two three...eight {nine} ten... |
+ } |
+ |
+ do_snippet_test $T.4.3 {one nine} 0 -5 { |
+ {one} two three four five...six seven eight {nine} ten |
+ } { |
+ {one} two three four five...seven eight {nine} ten eleven... |
+ } |
+ do_snippet_test $T.4.3 {one nineteen} 0 -5 { |
+ ...eighteen {nineteen} twenty {one} two... |
+ } |
+ do_snippet_test $T.4.4 {two nineteen} 0 -5 { |
+ ...eighteen {nineteen} twenty one {two}... |
+ } |
+ do_snippet_test $T.4.5 {three nineteen} 0 -5 { |
+ ...{nineteen} twenty one two {three}... |
+ } |
+ |
+ do_snippet_test $T.4.6 {four nineteen} 0 -5 { |
+ ...two three {four} five six...seventeen eighteen {nineteen} twenty one... |
+ } |
+ do_snippet_test $T.4.7 {four NEAR nineteen} 0 -5 { |
+ ...seventeen eighteen {nineteen} twenty one...two three {four} five six... |
+ } |
+ |
+ do_snippet_test $T.4.8 {four nineteen} 0 5 { |
+ ...three {four} five...eighteen {nineteen} twenty... |
+ } |
+ do_snippet_test $T.4.9 {four NEAR nineteen} 0 5 { |
+ ...eighteen {nineteen} twenty...three {four} five... |
+ } |
+ do_snippet_test $T.4.10 {four NEAR nineteen} 0 -5 { |
+ ...seventeen eighteen {nineteen} twenty one...two three {four} five six... |
+ } |
+ do_snippet_test $T.4.11 {four NOT (nineteen twentyone)} 0 5 { |
+ ...two three {four} five six... |
+ } { |
+ ...two three {four} five six... |
+ } |
+ do_snippet_test $T.4.12 {four OR nineteen NEAR twentyone} 0 5 { |
+ ...two three {four} five six... |
+ } { |
+ ...two three {four} five six... |
+ } |
+ |
+ do_test $T.5.1 { |
+ execsql { |
+ DROP TABLE IF EXISTS ft; |
+ CREATE VIRTUAL TABLE ft USING fts3(a, b, c); |
+ INSERT INTO ft VALUES( |
+ 'one two three four five', |
+ 'four five six seven eight', |
+ 'seven eight nine ten eleven' |
+ ); |
+ } |
+ } {} |
+ |
+ do_snippet_test $T.5.2 {five} -1 3 {...three four {five}} |
+ do_snippet_test $T.5.3 {five} 0 3 {...three four {five}} |
+ do_snippet_test $T.5.4 {five} 1 3 {four {five} six...} |
+ do_snippet_test $T.5.5 {five} 2 3 {seven eight nine...} |
+ |
+ do_test $T.5.6 { |
+ execsql { UPDATE ft SET b = NULL } |
+ } {} |
+ |
+ do_snippet_test $T.5.7 {five} -1 3 {...three four {five}} |
+ do_snippet_test $T.5.8 {five} 0 3 {...three four {five}} |
+ do_snippet_test $T.5.9 {five} 1 3 {} |
+ do_snippet_test $T.5.10 {five} 2 3 {seven eight nine...} |
+ |
+ do_snippet_test $T.5.11 {one "seven eight nine"} -1 -3 { |
+ {one} two three...{seven} {eight} {nine}... |
+ } |
+ |
+ do_test $T.6.1 { |
+ execsql { |
+ DROP TABLE IF EXISTS ft; |
+ CREATE VIRTUAL TABLE ft USING fts3(x); |
+ INSERT INTO ft VALUES($numbers); |
+ } |
+ } {} |
+ do_snippet_test $T.6.2 { |
+ one fifty onehundred onehundredfifty twohundredfifty threehundred |
+ } -1 4 { |
+ {one}...{fifty}...{onehundred}...{onehundredfifty}... |
+ } |
+ do_snippet_test $T.6.3 { |
+ one fifty onehundred onehundredfifty twohundredfifty threehundred |
+ } -1 -4 { |
+ {one} two three four...fortyeight fortynine {fifty} fiftyone...ninetyeight ninetynine {onehundred} onehundredone...onehundredfortyeight onehundredfortynine {onehundredfifty} onehundredfiftyone... |
+ } |
+ |
+ do_test $T.7.1 { |
+ execsql { |
+ BEGIN; |
+ DROP TABLE IF EXISTS ft; |
+ CREATE VIRTUAL TABLE ft USING fts3(x); |
+ } |
+ set testresults [list] |
+ for {set i 1} {$i < 150} {incr i} { |
+ set commas [string repeat , $i] |
+ execsql {INSERT INTO ft VALUES('one' || $commas || 'two')} |
+ lappend testresults "{one}$commas{two}" |
+ } |
+ execsql COMMIT |
+ } {} |
+ eval [list do_snippet_test $T.7.2 {one two} -1 3] $testresults |
+ |
+ ########################################################################## |
+ # Test the matchinfo function. |
+ # |
+ proc mit {blob} { |
+ set scan(littleEndian) i* |
+ set scan(bigEndian) I* |
+ binary scan $blob $scan($::tcl_platform(byteOrder)) r |
+ return $r |
+ } |
+ db func mit mit |
+ proc do_matchinfo_test {name expr args} { |
+ set res [list] |
+ foreach a $args { lappend res [normalize $a] } |
+ do_select_test $name { |
+ SELECT mit(matchinfo(ft)) FROM ft WHERE ft MATCH $expr |
+ } $res |
+ } |
+ do_test $T.8.1 { |
+ set ten {one two three four five six seven eight nine ten} |
+ execsql { |
+ DROP TABLE IF EXISTS ft; |
+ CREATE VIRTUAL TABLE ft USING fts3; |
+ INSERT INTO ft VALUES($ten); |
+ INSERT INTO ft VALUES($ten || ' ' || $ten); |
+ } |
+ } {} |
+ |
+ do_matchinfo_test $T.8.2 "one" {1 1 1 3 2} {1 1 2 3 2} |
+ do_matchinfo_test $T.8.3 "one NEAR/3 ten" {2 1 1 1 1 1 1 1} |
+ do_matchinfo_test $T.8.4 "five NEAR/4 ten" \ |
+ {2 1 1 3 2 1 3 2} {2 1 2 3 2 2 3 2} |
+ do_matchinfo_test $T.8.5 "six NEAR/3 ten NEAR/3 two" \ |
+ {3 1 1 1 1 1 1 1 1 1 1} |
+ do_matchinfo_test $T.8.6 "five NEAR/4 ten NEAR/3 two" \ |
+ {3 1 2 2 1 1 1 1 1 1 1} |
+ |
+ do_test $T.9.1 { |
+ execsql { |
+ DROP TABLE IF EXISTS ft; |
+ CREATE VIRTUAL TABLE ft USING fts3(x, y); |
+ } |
+ foreach n {1 2 3} { |
+ set v1 [lrange $numbers 0 [expr $n*100]] |
+ set v2 [string trim [string repeat "$numbers " $n]] |
+ set docid [expr $n * 1000000] |
+ execsql { INSERT INTO ft(docid, x, y) VALUES($docid, $v1, $v2) } |
+ } |
+ } {} |
+ do_matchinfo_test $T.9.2 {two*} \ |
+ { 1 2 1 105 3 101 606 3} \ |
+ { 1 2 3 105 3 202 606 3} \ |
+ { 1 2 101 105 3 303 606 3} |
+ |
+ do_matchinfo_test $T.9.4 {"one* two*"} \ |
+ { 1 2 1 5 3 2 12 3} \ |
+ { 1 2 2 5 3 4 12 3} \ |
+ { 1 2 2 5 3 6 12 3} |
+ |
+ do_matchinfo_test $T.9.5 {twohundredfifty} \ |
+ { 1 2 0 1 1 1 6 3} \ |
+ { 1 2 0 1 1 2 6 3} \ |
+ { 1 2 1 1 1 3 6 3} |
+ |
+ do_matchinfo_test $T.9.6 {"threehundred one"} \ |
+ { 1 2 0 0 0 1 3 2} \ |
+ { 1 2 0 0 0 2 3 2} |
+ |
+ do_matchinfo_test $T.9.7 {one OR fivehundred} \ |
+ { 2 2 1 3 3 1 6 3 0 0 0 0 0 0 } \ |
+ { 2 2 1 3 3 2 6 3 0 0 0 0 0 0 } \ |
+ { 2 2 1 3 3 3 6 3 0 0 0 0 0 0 } |
+ |
+ do_matchinfo_test $T.9.8 {two OR "threehundred one"} \ |
+ { 2 2 1 3 3 1 6 3 0 0 0 0 3 2 } \ |
+ { 2 2 1 3 3 2 6 3 0 0 0 1 3 2 } \ |
+ { 2 2 1 3 3 3 6 3 0 0 0 2 3 2 } |
+ |
+ do_select_test $T.9.9 { |
+ SELECT mit(matchinfo(ft)), mit(matchinfo(ft)) |
+ FROM ft WHERE ft MATCH 'two OR "threehundred one"' |
+ } [normalize { |
+ {2 2 1 3 3 1 6 3 0 0 0 0 3 2} |
+ {2 2 1 3 3 1 6 3 0 0 0 0 3 2} |
+ {2 2 1 3 3 2 6 3 0 0 0 1 3 2} |
+ {2 2 1 3 3 2 6 3 0 0 0 1 3 2} |
+ {2 2 1 3 3 3 6 3 0 0 0 2 3 2} |
+ {2 2 1 3 3 3 6 3 0 0 0 2 3 2} |
+ }] |
+ |
+ # EVIDENCE-OF: R-40630-02268 If used within a SELECT that uses the |
+ # "query by rowid" or "linear scan" strategies, then the snippet and |
+ # offsets both return an empty string, and the matchinfo function |
+ # returns a blob value zero bytes in size. |
+ # |
+ set r 1000000 ;# A rowid that exists in table ft |
+ do_select_test $T.10.0 { SELECT rowid FROM ft WHERE rowid = $r } $r |
+ do_select_test $T.10.1 { |
+ SELECT length(offsets(ft)), typeof(offsets(ft)) FROM ft; |
+ } {0 text 0 text 0 text} |
+ do_select_test $T.10.2 { |
+ SELECT length(offsets(ft)), typeof(offsets(ft)) FROM ft WHERE rowid = $r |
+ } {0 text} |
+ do_select_test $T.10.3 { |
+ SELECT length(snippet(ft)), typeof(snippet(ft)) FROM ft; |
+ } {0 text 0 text 0 text} |
+ do_select_test $T.10.4 { |
+ SELECT length(snippet(ft)), typeof(snippet(ft)) FROM ft WHERE rowid = $r; |
+ } {0 text} |
+ do_select_test $T.10.5 { |
+ SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft; |
+ } {0 blob 0 blob 0 blob} |
+ do_select_test $T.10.6 { |
+ SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft WHERE rowid = $r |
+ } {0 blob} |
+} |
+ |
+set sqlite_fts3_enable_parentheses 0 |
+finish_test |