123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494 |
- source [file join [file dirname [info script]] fts5_common.tcl]
- set testprefix fts5unicode2
- ifcapable !fts5 {
- finish_test
- return
- }
- proc do_unicode_token_test {tn input res} {
- uplevel [list do_test $tn [list \
- sqlite3_fts5_tokenize -subst db "unicode61 remove_diacritics 0" $input
- ] [list {*}$res]]
- }
- proc do_unicode_token_test2 {tn input res} {
- uplevel [list do_test $tn [list \
- sqlite3_fts5_tokenize -subst db "unicode61" $input
- ] [list {*}$res]]
- }
- proc do_unicode_token_test3 {tn args} {
- set tokenizer [concat unicode61 {*}[lrange $args 0 end-2]]
- set input [lindex $args end-1]
- set res [lindex $args end]
- uplevel [list do_test $tn [list \
- sqlite3_fts5_tokenize -subst db $tokenizer $input
- ] [list {*}$res]]
- }
- do_unicode_token_test 1.0 {a B c D} {a a b B c c d D}
- do_unicode_token_test 1.1 "\uC4 \uD6 \uDC" \
- "\uE4 \uC4 \uF6 \uD6 \uFC \uDC"
- do_unicode_token_test 1.2 "x\uC4x x\uD6x x\uDCx" \
- "x\uE4x x\uC4x x\uF6x x\uD6x x\uFCx x\uDCx"
- do_unicode_token_test 1.3 "\uDF" "\uDF \uDF"
- do_unicode_token_test 1.4 "\u1E9E" "\uDF \u1E9E"
- do_unicode_token_test 1.5 "The quick brown fox" {
- the The quick quick brown brown fox fox
- }
- do_unicode_token_test 1.6 "The\u00bfquick\u224ebrown\u2263fox" {
- the The quick quick brown brown fox fox
- }
- do_unicode_token_test2 1.7 {a B c D} {a a b B c c d D}
- do_unicode_token_test2 1.8 "\uC4 \uD6 \uDC" "a \uC4 o \uD6 u \uDC"
- do_unicode_token_test2 1.9 "x\uC4x x\uD6x x\uDCx" \
- "xax x\uC4x xox x\uD6x xux x\uDCx"
- do_unicode_token_test2 1.10 "xx\u0301xx" "xxxx xx\u301xx"
- do_unicode_token_test 1.11 "\u01c5" "\u01c6 \u01c5"
- do_unicode_token_test 1.12 "\u00C1abc\u00C2 \u00D1def\u00C3" \
- "\u00E1abc\u00E2 \u00C1abc\u00C2 \u00F1def\u00E3 \u00D1def\u00C3"
- do_unicode_token_test 1.13 "\u00A2abc\u00A3 \u00A4def\u00A5" \
- "abc abc def def"
- set docs [list {
- Enhance the INSERT syntax to allow multiple rows to be inserted via the
- VALUES clause.
- } {
- Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause.
- } {
- Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp().
- } {
- Added the sqlite3_db_readonly() interface.
- } {
- Added the SQLITE_FCNTL_PRAGMA file control, giving VFS implementations the
- ability to add new PRAGMA statements or to override built-in PRAGMAs.
- } {
- Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
- the same row that contains the maximum x value.
- } {
- Added support for the FTS4 languageid option.
- } {
- Documented support for the FTS4 content option. This feature has actually
- been in the code since version 3.7.9 but is only now considered to be
- officially supported.
- } {
- Pending statements no longer block ROLLBACK. Instead, the pending statement
- will return SQLITE_ABORT upon next access after the ROLLBACK.
- } {
- Improvements to the handling of CSV inputs in the command-line shell
- } {
- Fix a bug introduced in version 3.7.10 that might cause a LEFT JOIN to be
- incorrectly converted into an INNER JOIN if the WHERE clause indexable terms
- connected by OR.
- }]
- unset -nocomplain map
- set map(a) [list "\u00C4" "\u00E4"]
- set map(e) [list "\u00CB" "\u00EB"]
- set map(i) [list "\u00CF" "\u00EF"]
- set map(o) [list "\u00D6" "\u00F6"]
- set map(u) [list "\u00DC" "\u00FC"]
- set map(y) [list "\u0178" "\u00FF"]
- set map(h) [list "\u1E26" "\u1E27"]
- set map(w) [list "\u1E84" "\u1E85"]
- set map(x) [list "\u1E8C" "\u1E8D"]
- foreach k [array names map] {
- lappend mappings [string toupper $k] [lindex $map($k) 0]
- lappend mappings $k [lindex $map($k) 1]
- }
- proc mapdoc {doc} {
- set doc [regsub -all {[[:space:]]+} $doc " "]
- string map $::mappings [string trim $doc]
- }
- do_test 2.0 {
- execsql { CREATE VIRTUAL TABLE t2 USING fts5(tokenize=unicode61, x); }
- foreach doc $docs {
- set d [mapdoc $doc]
- execsql { INSERT INTO t2 VALUES($d) }
- }
- } {}
- do_test 2.1 {
- set q [mapdoc "row"]
- execsql { SELECT * FROM t2 WHERE t2 MATCH $q }
- } [list [mapdoc {
- Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
- the same row that contains the maximum x value.
- }]]
- foreach {tn query snippet} {
- 2 "row" {
- ...returns the value of y on the same [row] that contains
- the maximum x value.
- }
- 3 "ROW" {
- ...returns the value of y on the same [row] that contains
- the maximum x value.
- }
- 4 "rollback" {
- Pending statements no longer block [ROLLBACK]. Instead, the pending
- statement will return SQLITE_ABORT upon...
- }
- 5 "rOllback" {
- Pending statements no longer block [ROLLBACK]. Instead, the pending
- statement will return SQLITE_ABORT upon...
- }
- 6 "lang*" {
- Added support for the FTS4 [languageid] option.
- }
- } {
- do_test 2.$tn {
- set q [mapdoc $query]
- execsql {
- SELECT snippet(t2, -1, '[', ']', '...', 15) FROM t2 WHERE t2 MATCH $q
- }
- } [list [mapdoc $snippet]]
- }
- reset_db
- do_execsql_test 3.1 {
- CREATE VIRTUAL TABLE t1 USING fts5(tokenize=unicode61, x, y);
- INSERT INTO t1 VALUES(NULL, 'a b c');
- }
- do_execsql_test 3.2 {
- SELECT snippet(t1, -1, '[', ']', '...', 15) FROM t1 WHERE t1 MATCH 'b'
- } {{a [b] c}}
- do_execsql_test 3.3 {
- BEGIN;
- DELETE FROM t1;
- INSERT INTO t1 VALUES('b b b b b b b b b b b', 'b b b b b b b b b b b b b');
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 SELECT * FROM t1;
- INSERT INTO t1 VALUES('a b c', NULL);
- INSERT INTO t1 VALUES('a x c', NULL);
- COMMIT;
- }
- do_execsql_test 3.4 {
- SELECT * FROM t1 WHERE t1 MATCH 'a b';
- } {{a b c} {}}
- reset_db
- do_test 4.1 {
- set a "abc\uFFFEdef"
- set b "abc\uD800def"
- set c "\uFFFEdef"
- set d "\uD800def"
- execsql {
- CREATE VIRTUAL TABLE t1 USING fts5(tokenize=unicode61, x);
- INSERT INTO t1 VALUES($a);
- INSERT INTO t1 VALUES($b);
- INSERT INTO t1 VALUES($c);
- INSERT INTO t1 VALUES($d);
- }
- execsql "CREATE VIRTUAL TABLE t8 USING fts5(
- a, b, tokenize=\"unicode61 separators '\uFFFE\uD800\u00BF'\"
- )"
- } {}
- do_test 4.2 {
- set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}]
- set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}]
- set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
- set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
- execsql {
- INSERT INTO t1 VALUES($a);
- INSERT INTO t1 VALUES($b);
- INSERT INTO t1 VALUES($c);
- INSERT INTO t1 VALUES($d);
- }
- } {}
- do_test 4.3 {
- set a [binary format c* {0xF7 0xBF 0xBF 0xBF}]
- set b [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF}]
- set c [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF}]
- set d [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF}]
- execsql {
- INSERT INTO t1 VALUES($a);
- INSERT INTO t1 VALUES($b);
- INSERT INTO t1 VALUES($c);
- INSERT INTO t1 VALUES($d);
- }
- } {}
- do_test 4.4 {
- sqlite3_exec_hex db {
- CREATE VIRTUAL TABLE t9 USING fts5(a, b,
- tokenize="unicode61 separators '%C09004'"
- );
- INSERT INTO t9(a) VALUES('abc%88def %89ghi%90');
- }
- } {0 {}}
- do_unicode_token_test3 5.1 {tokenchars {}} {
- sqlite3_reset sqlite3_column_int
- } {
- sqlite3 sqlite3
- reset reset
- sqlite3 sqlite3
- column column
- int int
- }
- do_unicode_token_test3 5.2 {tokenchars _} {
- sqlite3_reset sqlite3_column_int
- } {
- sqlite3_reset sqlite3_reset
- sqlite3_column_int sqlite3_column_int
- }
- do_unicode_token_test3 5.3 {separators xyz} {
- Laotianxhorseyrunszfast
- } {
- laotian Laotian
- horse horse
- runs runs
- fast fast
- }
- do_unicode_token_test3 5.4 {tokenchars xyz} {
- Laotianxhorseyrunszfast
- } {
- laotianxhorseyrunszfast Laotianxhorseyrunszfast
- }
- do_unicode_token_test3 5.5 {tokenchars _} {separators zyx} {
- sqlite3_resetxsqlite3_column_intyhonda_phantom
- } {
- sqlite3_reset sqlite3_reset
- sqlite3_column_int sqlite3_column_int
- honda_phantom honda_phantom
- }
- do_unicode_token_test3 5.6 "separators \u05D1" "abc\u05D1def" {
- abc abc def def
- }
- do_unicode_token_test3 5.7 \
- "tokenchars \u2444\u2445" \
- "separators \u05D0\u05D1\u05D2" \
- "\u2444fre\u2445sh\u05D0water\u05D2fish.\u2445timer" \
- [list \
- \u2444fre\u2445sh \u2444fre\u2445sh \
- water water \
- fish fish \
- \u2445timer \u2445timer \
- ]
- do_unicode_token_test3 5.8 "separators \u0301" \
- "hello\u0301world \u0301helloworld" \
- "helloworld hello\u0301world helloworld helloworld"
- do_unicode_token_test3 5.9 "tokenchars \u0301" \
- "hello\u0301world \u0301helloworld" \
- "helloworld hello\u0301world helloworld helloworld"
- do_unicode_token_test3 5.10 "separators \u0301" \
- "remove_diacritics 0" \
- "hello\u0301world \u0301helloworld" \
- "hello\u0301world hello\u0301world helloworld helloworld"
- do_unicode_token_test3 5.11 "tokenchars \u0301" \
- "remove_diacritics 0" \
- "hello\u0301world \u0301helloworld" \
- "hello\u0301world hello\u0301world helloworld helloworld"
- proc do_tokenize {tokenizer txt} {
- set res [list]
- foreach {b c} [sqlite3_fts5_tokenize -subst db $tokenizer $txt] {
- lappend res $b
- }
- set res
- }
- proc do_isspace_test {tn tokenizer lCp} {
- set whitespace [format [string repeat %c [llength $lCp]] {*}$lCp]
- set txt "${whitespace}hello${whitespace}world${whitespace}"
- uplevel [list do_test $tn [list do_tokenize $tokenizer $txt] {hello world}]
- }
- set tokenizers [list unicode61]
- foreach T $tokenizers {
- do_isspace_test 6.$T.1 $T 32
- do_isspace_test 6.$T.2 $T 160
- do_isspace_test 6.$T.3 $T 5760
- do_isspace_test 6.$T.4 $T 6158
- do_isspace_test 6.$T.5 $T 8192
- do_isspace_test 6.$T.6 $T 8193
- do_isspace_test 6.$T.7 $T 8194
- do_isspace_test 6.$T.8 $T 8195
- do_isspace_test 6.$T.9 $T 8196
- do_isspace_test 6.$T.10 $T 8197
- do_isspace_test 6.$T.11 $T 8198
- do_isspace_test 6.$T.12 $T 8199
- do_isspace_test 6.$T.13 $T 8200
- do_isspace_test 6.$T.14 $T 8201
- do_isspace_test 6.$T.15 $T 8202
- do_isspace_test 6.$T.16 $T 8239
- do_isspace_test 6.$T.17 $T 8287
- do_isspace_test 6.$T.18 $T 12288
- do_isspace_test 6.$T.19 $T {32 160 5760 6158}
- do_isspace_test 6.$T.20 $T {8192 8193 8194 8195}
- do_isspace_test 6.$T.21 $T {8196 8197 8198 8199}
- do_isspace_test 6.$T.22 $T {8200 8201 8202 8239}
- do_isspace_test 6.$T.23 $T {8287 12288}
- }
- foreach {tn1 c} {
- 1 \ue000 2 \ue001 3 \uf000 4 \uf8fe 5 \uf8ff
- } {
- foreach {tn2 config res} {
- 1 "" "hello*world hello*world"
- 2 "separators *" "hello hello world world"
- } {
- set config [string map [list * $c] $config]
- set input [string map [list * $c] "hello*world"]
- set output [string map [list * $c] $res]
- do_unicode_token_test3 7.$tn1.$tn2 {*}$config $input $output
- }
- }
- do_execsql_test 8.1.1 "
- CREATE VIRTUAL TABLE t3 USING fts5(
- content, tokenize='unicode61 remove_diacritics 1'
- );
- INSERT INTO t3 VALUES('o');
- INSERT INTO t3 VALUES('a');
- INSERT INTO t3 VALUES('O');
- INSERT INTO t3 VALUES('A');
- INSERT INTO t3 VALUES('\xD6');
- INSERT INTO t3 VALUES('\xC4');
- INSERT INTO t3 VALUES('\xF6');
- INSERT INTO t3 VALUES('\xE4');
- "
- do_execsql_test 8.1.2 {
- SELECT rowid FROM t3 WHERE t3 MATCH 'o' ORDER BY rowid ASC;
- } {1 3 5 7}
- do_execsql_test 8.1.3 {
- SELECT rowid FROM t3 WHERE t3 MATCH 'a' ORDER BY rowid ASC;
- } {2 4 6 8}
- do_execsql_test 8.2.1 {
- CREATE VIRTUAL TABLE t4 USING fts5(
- content, tokenize='unicode61 remove_diacritics 0'
- );
- INSERT INTO t4 SELECT * FROM t3 ORDER BY rowid ASC;
- }
- do_execsql_test 8.2.2 {
- SELECT rowid FROM t4 WHERE t4 MATCH 'o' ORDER BY rowid ASC;
- } {1 3}
- do_execsql_test 8.2.3 {
- SELECT rowid FROM t4 WHERE t4 MATCH 'a' ORDER BY rowid ASC;
- } {2 4}
- foreach {tn val bErr} {
- 1 0 0
- 2 1 0
- 3 2 0
- 4 3 1
- 5 11 1
- } {
- reset_db
- set aRes(0) {0 {}}
- set aRes(1) {1 {error in tokenizer constructor}}
- set res $aRes($bErr)
- do_catchsql_test 9.1.$tn "
- CREATE VIRTUAL TABLE bl USING fts5(
- s, tokenize='trigram remove_diacritics $val'
- );
- " $res
- }
- finish_test
|