OLD | NEW |
1 This patch removes the usage of tolower() in fts code, which is not locale | 1 diff --git a/third_party/sqlite/src/ext/fts1/fts1.c b/third_party/sqlite/src/ext
/fts1/fts1.c |
2 neutral and causes problem in some locales such as Turkish. | 2 index 8ae8df4..d5d00b9 100644 |
3 See http://crbug.com/15261 for details. | 3 --- a/third_party/sqlite/src/ext/fts1/fts1.c |
4 An upstream ticket was also created for this issue: | 4 +++ b/third_party/sqlite/src/ext/fts1/fts1.c |
5 http://www.sqlite.org/src/tktview/991789d9f3136a0460dc83a33e815c1aa9757c26 | 5 @@ -208,7 +208,7 @@ static int safe_isspace(char c){ |
6 | |
7 Contains backport for upstream http://www.sqlite.org/src/ci/b8b465ed2c. | |
8 | |
9 Index: ext/fts3/fts3.c | |
10 =================================================================== | |
11 --- ext/fts3/fts3.c» 2009-09-04 13:37:41.000000000 -0700 | |
12 +++ ext/fts3/fts3.c» 2009-09-14 18:17:45.000000000 -0700 | |
13 @@ -326,7 +326,7 @@ | |
14 return (c&0x80)==0 ? isspace(c) : 0; | 6 return (c&0x80)==0 ? isspace(c) : 0; |
15 } | 7 } |
16 static int safe_tolower(char c){ | 8 static int safe_tolower(char c){ |
17 - return (c&0x80)==0 ? tolower(c) : c; | 9 - return (c&0x80)==0 ? tolower(c) : c; |
18 + return (c>='A' && c<='Z') ? (c-'A'+'a') : c; | 10 + return (c>='A' && c<='Z') ? (c-'A'+'a') : c; |
19 } | 11 } |
20 static int safe_isalnum(char c){ | 12 static int safe_isalnum(char c){ |
21 return (c&0x80)==0 ? isalnum(c) : 0; | 13 return (c&0x80)==0 ? isalnum(c) : 0; |
22 Index: ext/fts3/fts3_expr.c | 14 diff --git a/third_party/sqlite/src/ext/fts1/fts1_tokenizer1.c b/third_party/sql
ite/src/ext/fts1/fts1_tokenizer1.c |
23 =================================================================== | 15 index f58fba8..90774da 100644 |
24 --- ext/fts3/fts3_expr.c | 16 --- a/third_party/sqlite/src/ext/fts1/fts1_tokenizer1.c |
25 +++ ext/fts3/fts3_expr.c | 17 +++ b/third_party/sqlite/src/ext/fts1/fts1_tokenizer1.c |
26 @@ -58,7 +58,6 @@ int sqlite3_fts3_enable_parentheses = 0; | 18 @@ -182,7 +182,7 @@ static int simpleNext( |
27 | |
28 #include "fts3_expr.h" | |
29 #include "sqlite3.h" | |
30 -#include <ctype.h> | |
31 #include <string.h> | |
32 #include <assert.h> | |
33 | |
34 @@ -84,7 +83,7 @@ struct ParseContext { | |
35 ** negative values). | |
36 */ | |
37 static int fts3isspace(char c){ | |
38 - return (c&0x80)==0 ? isspace(c) : 0; | |
39 + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; | |
40 } | |
41 | |
42 /* | |
43 Index: ext/fts3/fts3_porter.c | |
44 =================================================================== | |
45 --- ext/fts3/fts3_porter.c | |
46 +++ ext/fts3/fts3_porter.c | |
47 @@ -29,7 +29,6 @@ | |
48 #include <stdlib.h> | |
49 #include <stdio.h> | |
50 #include <string.h> | |
51 -#include <ctype.h> | |
52 | |
53 #include "fts3_tokenizer.h" | |
54 | |
55 Index: ext/fts3/fts3_tokenizer1.c | |
56 =================================================================== | |
57 --- ext/fts3/fts3_tokenizer1.c | |
58 +++ ext/fts3/fts3_tokenizer1.c | |
59 @@ -29,7 +29,6 @@ | |
60 #include <stdlib.h> | |
61 #include <stdio.h> | |
62 #include <string.h> | |
63 -#include <ctype.h> | |
64 | |
65 #include "fts3_tokenizer.h" | |
66 | |
67 @@ -55,6 +54,9 @@ static const sqlite3_tokenizer_module simpleTokenizerModule; | |
68 static int simpleDelim(simple_tokenizer *t, unsigned char c){ | |
69 return c<0x80 && t->delim[c]; | |
70 } | |
71 +static int fts3_isalnum(int x){ | |
72 + return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); | |
73 +} | |
74 | |
75 /* | |
76 ** Create a new tokenizer instance. | |
77 @@ -89,7 +91,7 @@ static int simpleCreate( | |
78 /* Mark non-alphanumeric ASCII characters as delimiters */ | |
79 int i; | |
80 for(i=1; i<0x80; i++){ | |
81 - t->delim[i] = !isalnum(i); | |
82 + t->delim[i] = !fts3_isalnum(i); | |
83 } | |
84 } | |
85 | |
86 @@ -191,7 +193,7 @@ static int simpleNext( | |
87 ** case-insensitivity. | 19 ** case-insensitivity. |
88 */ | 20 */ |
89 unsigned char ch = p[iStartOffset+i]; | 21 unsigned char ch = p[iStartOffset+i]; |
90 - c->pToken[i] = ch<0x80 ? tolower(ch) : ch; | 22 - c->pToken[i] = ch<0x80 ? tolower(ch) : ch; |
91 + c->pToken[i] = (ch>='A' && ch<='Z') ? ch-'A'+'a' : ch; | 23 + c->pToken[i] = (ch>='A' && ch<='Z') ? (ch-'A'+'a') : ch; |
92 } | 24 } |
93 *ppToken = c->pToken; | 25 *ppToken = c->pToken; |
94 *pnBytes = n; | 26 *pnBytes = n; |
95 Index: ext/fts1/simple_tokenizer.c | 27 diff --git a/third_party/sqlite/src/ext/fts1/simple_tokenizer.c b/third_party/sq
lite/src/ext/fts1/simple_tokenizer.c |
96 =================================================================== | 28 index d00a770..9a52bd1 100644 |
97 --- ext/fts1/simple_tokenizer.c»2009-09-03 13:32:06.000000000 -0700 | 29 --- a/third_party/sqlite/src/ext/fts1/simple_tokenizer.c |
98 +++ ext/fts1/simple_tokenizer.c»2009-09-02 11:40:21.000000000 -0700 | 30 +++ b/third_party/sqlite/src/ext/fts1/simple_tokenizer.c |
99 @@ -138,7 +138,7 @@ | 31 @@ -138,7 +138,7 @@ static int simpleNext( |
100 ** case-insensitivity. | 32 ** case-insensitivity. |
101 */ | 33 */ |
102 char ch = c->pCurrent[ii]; | 34 char ch = c->pCurrent[ii]; |
103 - c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch; | 35 - c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch; |
104 + c->zToken[ii] = ((ch>='A' && ch<='Z') ? (ch-'A'+'a') : ch); | 36 + c->zToken[ii] = ((ch>='A' && ch<='Z') ? (ch-'A'+'a') : ch); |
105 } | 37 } |
106 c->zToken[n] = '\0'; | 38 c->zToken[n] = '\0'; |
107 *ppToken = c->zToken; | 39 *ppToken = c->zToken; |
108 Index: ext/fts1/fts1_tokenizer1.c | |
109 =================================================================== | |
110 --- ext/fts1/fts1_tokenizer1.c 2009-09-03 13:32:06.000000000 -0700 | |
111 +++ ext/fts1/fts1_tokenizer1.c 2009-09-02 11:40:21.000000000 -0700 | |
112 @@ -182,7 +182,7 @@ | |
113 ** case-insensitivity. | |
114 */ | |
115 unsigned char ch = p[iStartOffset+i]; | |
116 - c->pToken[i] = ch<0x80 ? tolower(ch) : ch; | |
117 + c->pToken[i] = (ch>='A' && ch<='Z') ? (ch-'A'+'a') : ch; | |
118 } | |
119 *ppToken = c->pToken; | |
120 *pnBytes = n; | |
121 Index: ext/fts1/fts1.c | |
122 =================================================================== | |
123 --- ext/fts1/fts1.c 2009-09-04 13:37:41.000000000 -0700 | |
124 +++ ext/fts1/fts1.c 2009-09-14 18:16:55.000000000 -0700 | |
125 @@ -208,7 +208,7 @@ | |
126 return (c&0x80)==0 ? isspace(c) : 0; | |
127 } | |
128 static int safe_tolower(char c){ | |
129 - return (c&0x80)==0 ? tolower(c) : c; | |
130 + return (c>='A' && c<='Z') ? (c-'A'+'a') : c; | |
131 } | |
132 static int safe_isalnum(char c){ | |
133 return (c&0x80)==0 ? isalnum(c) : 0; | |
134 Index: ext/fts2/fts2.c | |
135 =================================================================== | |
136 --- ext/fts2/fts2.c 2009-09-04 13:37:41.000000000 -0700 | |
137 +++ ext/fts2/fts2.c 2009-09-14 18:17:02.000000000 -0700 | |
138 @@ -372,7 +372,7 @@ | |
139 return (c&0x80)==0 ? isspace(c) : 0; | |
140 } | |
141 static int safe_tolower(char c){ | |
142 - return (c&0x80)==0 ? tolower(c) : c; | |
143 + return (c>='A' && c<='Z') ? (c-'A'+'a') : c; | |
144 } | |
145 static int safe_isalnum(char c){ | |
146 return (c&0x80)==0 ? isalnum(c) : 0; | |
147 Index: ext/fts2/fts2_tokenizer1.c | |
148 =================================================================== | |
149 --- ext/fts2/fts2_tokenizer1.c 2009-09-03 13:32:06.000000000 -0700 | |
150 +++ ext/fts2/fts2_tokenizer1.c 2009-09-02 11:40:21.000000000 -0700 | |
151 @@ -191,7 +191,7 @@ | |
152 ** case-insensitivity. | |
153 */ | |
154 unsigned char ch = p[iStartOffset+i]; | |
155 - c->pToken[i] = ch<0x80 ? tolower(ch) : ch; | |
156 + c->pToken[i] = (ch>='A' && ch<='Z') ? (ch-'A'+'a') : ch; | |
157 } | |
158 *ppToken = c->pToken; | |
159 *pnBytes = n; | |
OLD | NEW |