OLD | NEW |
1 /* | 1 /* |
2 ** 2014 May 31 | 2 ** 2014 May 31 |
3 ** | 3 ** |
4 ** The author disclaims copyright to this source code. In place of | 4 ** The author disclaims copyright to this source code. In place of |
5 ** a legal notice, here is a blessing: | 5 ** a legal notice, here is a blessing: |
6 ** | 6 ** |
7 ** May you do good and not evil. | 7 ** May you do good and not evil. |
8 ** May you find forgiveness for yourself and forgive others. | 8 ** May you find forgiveness for yourself and forgive others. |
9 ** May you share freely, never taking more than you give. | 9 ** May you share freely, never taking more than you give. |
10 ** | 10 ** |
(...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
151 int tflags, /* Mask of FTS5_TOKEN_* flags */ | 151 int tflags, /* Mask of FTS5_TOKEN_* flags */ |
152 const char *pToken, /* Buffer containing token */ | 152 const char *pToken, /* Buffer containing token */ |
153 int nToken, /* Size of token in bytes */ | 153 int nToken, /* Size of token in bytes */ |
154 int iStartOff, /* Start offset of token */ | 154 int iStartOff, /* Start offset of token */ |
155 int iEndOff /* End offset of token */ | 155 int iEndOff /* End offset of token */ |
156 ){ | 156 ){ |
157 HighlightContext *p = (HighlightContext*)pContext; | 157 HighlightContext *p = (HighlightContext*)pContext; |
158 int rc = SQLITE_OK; | 158 int rc = SQLITE_OK; |
159 int iPos; | 159 int iPos; |
160 | 160 |
| 161 UNUSED_PARAM2(pToken, nToken); |
| 162 |
161 if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; | 163 if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; |
162 iPos = p->iPos++; | 164 iPos = p->iPos++; |
163 | 165 |
164 if( p->iRangeEnd>0 ){ | 166 if( p->iRangeEnd>0 ){ |
165 if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; | 167 if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; |
166 if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; | 168 if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; |
167 } | 169 } |
168 | 170 |
169 if( iPos==p->iter.iStart ){ | 171 if( iPos==p->iter.iStart ){ |
170 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff); | 172 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff); |
171 fts5HighlightAppend(&rc, p, p->zOpen, -1); | 173 fts5HighlightAppend(&rc, p, p->zOpen, -1); |
172 p->iOff = iStartOff; | 174 p->iOff = iStartOff; |
173 } | 175 } |
174 | 176 |
175 if( iPos==p->iter.iEnd ){ | 177 if( iPos==p->iter.iEnd ){ |
176 if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){ | 178 if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){ |
177 fts5HighlightAppend(&rc, p, p->zOpen, -1); | 179 fts5HighlightAppend(&rc, p, p->zOpen, -1); |
178 } | 180 } |
179 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); | 181 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); |
180 fts5HighlightAppend(&rc, p, p->zClose, -1); | 182 fts5HighlightAppend(&rc, p, p->zClose, -1); |
181 p->iOff = iEndOff; | 183 p->iOff = iEndOff; |
182 if( rc==SQLITE_OK ){ | 184 if( rc==SQLITE_OK ){ |
183 rc = fts5CInstIterNext(&p->iter); | 185 rc = fts5CInstIterNext(&p->iter); |
184 } | 186 } |
185 } | 187 } |
186 | 188 |
187 if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ | 189 if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ |
188 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); | 190 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); |
189 p->iOff = iEndOff; | 191 p->iOff = iEndOff; |
190 if( iPos<p->iter.iEnd ){ | 192 if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){ |
191 fts5HighlightAppend(&rc, p, p->zClose, -1); | 193 fts5HighlightAppend(&rc, p, p->zClose, -1); |
192 } | 194 } |
193 } | 195 } |
194 | 196 |
195 return rc; | 197 return rc; |
196 } | 198 } |
197 | 199 |
198 /* | 200 /* |
199 ** Implementation of highlight() function. | 201 ** Implementation of highlight() function. |
200 */ | 202 */ |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
238 } | 240 } |
239 if( rc!=SQLITE_OK ){ | 241 if( rc!=SQLITE_OK ){ |
240 sqlite3_result_error_code(pCtx, rc); | 242 sqlite3_result_error_code(pCtx, rc); |
241 } | 243 } |
242 } | 244 } |
243 /* | 245 /* |
244 ** End of highlight() implementation. | 246 ** End of highlight() implementation. |
245 **************************************************************************/ | 247 **************************************************************************/ |
246 | 248 |
247 /* | 249 /* |
| 250 ** Context object passed to the fts5SentenceFinderCb() function. |
| 251 */ |
| 252 typedef struct Fts5SFinder Fts5SFinder; |
| 253 struct Fts5SFinder { |
| 254 int iPos; /* Current token position */ |
| 255 int nFirstAlloc; /* Allocated size of aFirst[] */ |
| 256 int nFirst; /* Number of entries in aFirst[] */ |
| 257 int *aFirst; /* Array of first token in each sentence */ |
| 258 const char *zDoc; /* Document being tokenized */ |
| 259 }; |
| 260 |
| 261 /* |
| 262 ** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if |
| 263 ** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an |
| 264 ** error occurs. |
| 265 */ |
| 266 static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){ |
| 267 if( p->nFirstAlloc==p->nFirst ){ |
| 268 int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64; |
| 269 int *aNew; |
| 270 |
| 271 aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int)); |
| 272 if( aNew==0 ) return SQLITE_NOMEM; |
| 273 p->aFirst = aNew; |
| 274 p->nFirstAlloc = nNew; |
| 275 } |
| 276 p->aFirst[p->nFirst++] = iAdd; |
| 277 return SQLITE_OK; |
| 278 } |
| 279 |
| 280 /* |
| 281 ** This function is an xTokenize() callback used by the auxiliary snippet() |
| 282 ** function. Its job is to identify tokens that are the first in a sentence. |
| 283 ** For each such token, an entry is added to the SFinder.aFirst[] array. |
| 284 */ |
| 285 static int fts5SentenceFinderCb( |
| 286 void *pContext, /* Pointer to HighlightContext object */ |
| 287 int tflags, /* Mask of FTS5_TOKEN_* flags */ |
| 288 const char *pToken, /* Buffer containing token */ |
| 289 int nToken, /* Size of token in bytes */ |
| 290 int iStartOff, /* Start offset of token */ |
| 291 int iEndOff /* End offset of token */ |
| 292 ){ |
| 293 int rc = SQLITE_OK; |
| 294 |
| 295 UNUSED_PARAM2(pToken, nToken); |
| 296 UNUSED_PARAM(iEndOff); |
| 297 |
| 298 if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ |
| 299 Fts5SFinder *p = (Fts5SFinder*)pContext; |
| 300 if( p->iPos>0 ){ |
| 301 int i; |
| 302 char c = 0; |
| 303 for(i=iStartOff-1; i>=0; i--){ |
| 304 c = p->zDoc[i]; |
| 305 if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break; |
| 306 } |
| 307 if( i!=iStartOff-1 && (c=='.' || c==':') ){ |
| 308 rc = fts5SentenceFinderAdd(p, p->iPos); |
| 309 } |
| 310 }else{ |
| 311 rc = fts5SentenceFinderAdd(p, 0); |
| 312 } |
| 313 p->iPos++; |
| 314 } |
| 315 return rc; |
| 316 } |
| 317 |
| 318 static int fts5SnippetScore( |
| 319 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ |
| 320 Fts5Context *pFts, /* First arg to pass to pApi functions */ |
| 321 int nDocsize, /* Size of column in tokens */ |
| 322 unsigned char *aSeen, /* Array with one element per query phrase */ |
| 323 int iCol, /* Column to score */ |
| 324 int iPos, /* Starting offset to score */ |
| 325 int nToken, /* Max tokens per snippet */ |
| 326 int *pnScore, /* OUT: Score */ |
| 327 int *piPos /* OUT: Adjusted offset */ |
| 328 ){ |
| 329 int rc; |
| 330 int i; |
| 331 int ip = 0; |
| 332 int ic = 0; |
| 333 int iOff = 0; |
| 334 int iFirst = -1; |
| 335 int nInst; |
| 336 int nScore = 0; |
| 337 int iLast = 0; |
| 338 |
| 339 rc = pApi->xInstCount(pFts, &nInst); |
| 340 for(i=0; i<nInst && rc==SQLITE_OK; i++){ |
| 341 rc = pApi->xInst(pFts, i, &ip, &ic, &iOff); |
| 342 if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){ |
| 343 nScore += (aSeen[ip] ? 1 : 1000); |
| 344 aSeen[ip] = 1; |
| 345 if( iFirst<0 ) iFirst = iOff; |
| 346 iLast = iOff + pApi->xPhraseSize(pFts, ip); |
| 347 } |
| 348 } |
| 349 |
| 350 *pnScore = nScore; |
| 351 if( piPos ){ |
| 352 int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2; |
| 353 if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken; |
| 354 if( iAdj<0 ) iAdj = 0; |
| 355 *piPos = iAdj; |
| 356 } |
| 357 |
| 358 return rc; |
| 359 } |
| 360 |
| 361 /* |
248 ** Implementation of snippet() function. | 362 ** Implementation of snippet() function. |
249 */ | 363 */ |
250 static void fts5SnippetFunction( | 364 static void fts5SnippetFunction( |
251 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ | 365 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ |
252 Fts5Context *pFts, /* First arg to pass to pApi functions */ | 366 Fts5Context *pFts, /* First arg to pass to pApi functions */ |
253 sqlite3_context *pCtx, /* Context for returning result/error */ | 367 sqlite3_context *pCtx, /* Context for returning result/error */ |
254 int nVal, /* Number of values in apVal[] array */ | 368 int nVal, /* Number of values in apVal[] array */ |
255 sqlite3_value **apVal /* Array of trailing arguments */ | 369 sqlite3_value **apVal /* Array of trailing arguments */ |
256 ){ | 370 ){ |
257 HighlightContext ctx; | 371 HighlightContext ctx; |
258 int rc = SQLITE_OK; /* Return code */ | 372 int rc = SQLITE_OK; /* Return code */ |
259 int iCol; /* 1st argument to snippet() */ | 373 int iCol; /* 1st argument to snippet() */ |
260 const char *zEllips; /* 4th argument to snippet() */ | 374 const char *zEllips; /* 4th argument to snippet() */ |
261 int nToken; /* 5th argument to snippet() */ | 375 int nToken; /* 5th argument to snippet() */ |
262 int nInst = 0; /* Number of instance matches this row */ | 376 int nInst = 0; /* Number of instance matches this row */ |
263 int i; /* Used to iterate through instances */ | 377 int i; /* Used to iterate through instances */ |
264 int nPhrase; /* Number of phrases in query */ | 378 int nPhrase; /* Number of phrases in query */ |
265 unsigned char *aSeen; /* Array of "seen instance" flags */ | 379 unsigned char *aSeen; /* Array of "seen instance" flags */ |
266 int iBestCol; /* Column containing best snippet */ | 380 int iBestCol; /* Column containing best snippet */ |
267 int iBestStart = 0; /* First token of best snippet */ | 381 int iBestStart = 0; /* First token of best snippet */ |
268 int iBestLast; /* Last token of best snippet */ | |
269 int nBestScore = 0; /* Score of best snippet */ | 382 int nBestScore = 0; /* Score of best snippet */ |
270 int nColSize = 0; /* Total size of iBestCol in tokens */ | 383 int nColSize = 0; /* Total size of iBestCol in tokens */ |
| 384 Fts5SFinder sFinder; /* Used to find the beginnings of sentences */ |
| 385 int nCol; |
271 | 386 |
272 if( nVal!=5 ){ | 387 if( nVal!=5 ){ |
273 const char *zErr = "wrong number of arguments to function snippet()"; | 388 const char *zErr = "wrong number of arguments to function snippet()"; |
274 sqlite3_result_error(pCtx, zErr, -1); | 389 sqlite3_result_error(pCtx, zErr, -1); |
275 return; | 390 return; |
276 } | 391 } |
277 | 392 |
| 393 nCol = pApi->xColumnCount(pFts); |
278 memset(&ctx, 0, sizeof(HighlightContext)); | 394 memset(&ctx, 0, sizeof(HighlightContext)); |
279 iCol = sqlite3_value_int(apVal[0]); | 395 iCol = sqlite3_value_int(apVal[0]); |
280 ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); | 396 ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); |
281 ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); | 397 ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); |
282 zEllips = (const char*)sqlite3_value_text(apVal[3]); | 398 zEllips = (const char*)sqlite3_value_text(apVal[3]); |
283 nToken = sqlite3_value_int(apVal[4]); | 399 nToken = sqlite3_value_int(apVal[4]); |
284 iBestLast = nToken-1; | |
285 | 400 |
286 iBestCol = (iCol>=0 ? iCol : 0); | 401 iBestCol = (iCol>=0 ? iCol : 0); |
287 nPhrase = pApi->xPhraseCount(pFts); | 402 nPhrase = pApi->xPhraseCount(pFts); |
288 aSeen = sqlite3_malloc(nPhrase); | 403 aSeen = sqlite3_malloc(nPhrase); |
289 if( aSeen==0 ){ | 404 if( aSeen==0 ){ |
290 rc = SQLITE_NOMEM; | 405 rc = SQLITE_NOMEM; |
291 } | 406 } |
292 | |
293 if( rc==SQLITE_OK ){ | 407 if( rc==SQLITE_OK ){ |
294 rc = pApi->xInstCount(pFts, &nInst); | 408 rc = pApi->xInstCount(pFts, &nInst); |
295 } | 409 } |
296 for(i=0; rc==SQLITE_OK && i<nInst; i++){ | |
297 int ip, iSnippetCol, iStart; | |
298 memset(aSeen, 0, nPhrase); | |
299 rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart); | |
300 if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){ | |
301 int nScore = 1000; | |
302 int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip); | |
303 int j; | |
304 aSeen[ip] = 1; | |
305 | 410 |
306 for(j=i+1; rc==SQLITE_OK && j<nInst; j++){ | 411 memset(&sFinder, 0, sizeof(Fts5SFinder)); |
307 int ic; int io; int iFinal; | 412 for(i=0; i<nCol; i++){ |
308 rc = pApi->xInst(pFts, j, &ip, &ic, &io); | 413 if( iCol<0 || iCol==i ){ |
309 iFinal = io + pApi->xPhraseSize(pFts, ip) - 1; | 414 int nDoc; |
310 if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){ | 415 int nDocsize; |
311 nScore += aSeen[ip] ? 1000 : 1; | 416 int ii; |
312 aSeen[ip] = 1; | 417 sFinder.iPos = 0; |
313 if( iFinal>iLast ) iLast = iFinal; | 418 sFinder.nFirst = 0; |
| 419 rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc); |
| 420 if( rc!=SQLITE_OK ) break; |
| 421 rc = pApi->xTokenize(pFts, |
| 422 sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb |
| 423 ); |
| 424 if( rc!=SQLITE_OK ) break; |
| 425 rc = pApi->xColumnSize(pFts, i, &nDocsize); |
| 426 if( rc!=SQLITE_OK ) break; |
| 427 |
| 428 for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){ |
| 429 int ip, ic, io; |
| 430 int iAdj; |
| 431 int nScore; |
| 432 int jj; |
| 433 |
| 434 rc = pApi->xInst(pFts, ii, &ip, &ic, &io); |
| 435 if( ic!=i || rc!=SQLITE_OK ) continue; |
| 436 memset(aSeen, 0, nPhrase); |
| 437 rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, |
| 438 io, nToken, &nScore, &iAdj |
| 439 ); |
| 440 if( rc==SQLITE_OK && nScore>nBestScore ){ |
| 441 nBestScore = nScore; |
| 442 iBestCol = i; |
| 443 iBestStart = iAdj; |
| 444 nColSize = nDocsize; |
314 } | 445 } |
315 } | |
316 | 446 |
317 if( rc==SQLITE_OK && nScore>nBestScore ){ | 447 if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){ |
318 iBestCol = iSnippetCol; | 448 for(jj=0; jj<(sFinder.nFirst-1); jj++){ |
319 iBestStart = iStart; | 449 if( sFinder.aFirst[jj+1]>io ) break; |
320 iBestLast = iLast; | 450 } |
321 nBestScore = nScore; | 451 |
| 452 if( sFinder.aFirst[jj]<io ){ |
| 453 memset(aSeen, 0, nPhrase); |
| 454 rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, |
| 455 sFinder.aFirst[jj], nToken, &nScore, 0 |
| 456 ); |
| 457 |
| 458 nScore += (sFinder.aFirst[jj]==0 ? 120 : 100); |
| 459 if( rc==SQLITE_OK && nScore>nBestScore ){ |
| 460 nBestScore = nScore; |
| 461 iBestCol = i; |
| 462 iBestStart = sFinder.aFirst[jj]; |
| 463 nColSize = nDocsize; |
| 464 } |
| 465 } |
| 466 } |
322 } | 467 } |
323 } | 468 } |
324 } | 469 } |
325 | 470 |
326 if( rc==SQLITE_OK ){ | 471 if( rc==SQLITE_OK ){ |
| 472 rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); |
| 473 } |
| 474 if( rc==SQLITE_OK && nColSize==0 ){ |
327 rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); | 475 rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); |
328 } | 476 } |
329 if( rc==SQLITE_OK ){ | |
330 rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); | |
331 } | |
332 if( ctx.zIn ){ | 477 if( ctx.zIn ){ |
333 if( rc==SQLITE_OK ){ | 478 if( rc==SQLITE_OK ){ |
334 rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); | 479 rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); |
335 } | 480 } |
336 | 481 |
337 if( (iBestStart+nToken-1)>iBestLast ){ | |
338 iBestStart -= (iBestStart+nToken-1-iBestLast) / 2; | |
339 } | |
340 if( iBestStart+nToken>nColSize ){ | |
341 iBestStart = nColSize - nToken; | |
342 } | |
343 if( iBestStart<0 ) iBestStart = 0; | |
344 | |
345 ctx.iRangeStart = iBestStart; | 482 ctx.iRangeStart = iBestStart; |
346 ctx.iRangeEnd = iBestStart + nToken - 1; | 483 ctx.iRangeEnd = iBestStart + nToken - 1; |
347 | 484 |
348 if( iBestStart>0 ){ | 485 if( iBestStart>0 ){ |
349 fts5HighlightAppend(&rc, &ctx, zEllips, -1); | 486 fts5HighlightAppend(&rc, &ctx, zEllips, -1); |
350 } | 487 } |
| 488 |
| 489 /* Advance iterator ctx.iter so that it points to the first coalesced |
| 490 ** phrase instance at or following position iBestStart. */ |
| 491 while( ctx.iter.iStart>=0 && ctx.iter.iStart<iBestStart && rc==SQLITE_OK ){ |
| 492 rc = fts5CInstIterNext(&ctx.iter); |
| 493 } |
| 494 |
351 if( rc==SQLITE_OK ){ | 495 if( rc==SQLITE_OK ){ |
352 rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); | 496 rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); |
353 } | 497 } |
354 if( ctx.iRangeEnd>=(nColSize-1) ){ | 498 if( ctx.iRangeEnd>=(nColSize-1) ){ |
355 fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); | 499 fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); |
356 }else{ | 500 }else{ |
357 fts5HighlightAppend(&rc, &ctx, zEllips, -1); | 501 fts5HighlightAppend(&rc, &ctx, zEllips, -1); |
358 } | 502 } |
359 | |
360 if( rc==SQLITE_OK ){ | |
361 sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); | |
362 }else{ | |
363 sqlite3_result_error_code(pCtx, rc); | |
364 } | |
365 sqlite3_free(ctx.zOut); | |
366 } | 503 } |
| 504 if( rc==SQLITE_OK ){ |
| 505 sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); |
| 506 }else{ |
| 507 sqlite3_result_error_code(pCtx, rc); |
| 508 } |
| 509 sqlite3_free(ctx.zOut); |
367 sqlite3_free(aSeen); | 510 sqlite3_free(aSeen); |
| 511 sqlite3_free(sFinder.aFirst); |
368 } | 512 } |
369 | 513 |
370 /************************************************************************/ | 514 /************************************************************************/ |
371 | 515 |
372 /* | 516 /* |
373 ** The first time the bm25() function is called for a query, an instance | 517 ** The first time the bm25() function is called for a query, an instance |
374 ** of the following structure is allocated and populated. | 518 ** of the following structure is allocated and populated. |
375 */ | 519 */ |
376 typedef struct Fts5Bm25Data Fts5Bm25Data; | 520 typedef struct Fts5Bm25Data Fts5Bm25Data; |
377 struct Fts5Bm25Data { | 521 struct Fts5Bm25Data { |
378 int nPhrase; /* Number of phrases in query */ | 522 int nPhrase; /* Number of phrases in query */ |
379 double avgdl; /* Average number of tokens in each row */ | 523 double avgdl; /* Average number of tokens in each row */ |
380 double *aIDF; /* IDF for each phrase */ | 524 double *aIDF; /* IDF for each phrase */ |
381 double *aFreq; /* Array used to calculate phrase freq. */ | 525 double *aFreq; /* Array used to calculate phrase freq. */ |
382 }; | 526 }; |
383 | 527 |
384 /* | 528 /* |
385 ** Callback used by fts5Bm25GetData() to count the number of rows in the | 529 ** Callback used by fts5Bm25GetData() to count the number of rows in the |
386 ** table matched by each individual phrase within the query. | 530 ** table matched by each individual phrase within the query. |
387 */ | 531 */ |
388 static int fts5CountCb( | 532 static int fts5CountCb( |
389 const Fts5ExtensionApi *pApi, | 533 const Fts5ExtensionApi *pApi, |
390 Fts5Context *pFts, | 534 Fts5Context *pFts, |
391 void *pUserData /* Pointer to sqlite3_int64 variable */ | 535 void *pUserData /* Pointer to sqlite3_int64 variable */ |
392 ){ | 536 ){ |
393 sqlite3_int64 *pn = (sqlite3_int64*)pUserData; | 537 sqlite3_int64 *pn = (sqlite3_int64*)pUserData; |
| 538 UNUSED_PARAM2(pApi, pFts); |
394 (*pn)++; | 539 (*pn)++; |
395 return SQLITE_OK; | 540 return SQLITE_OK; |
396 } | 541 } |
397 | 542 |
398 /* | 543 /* |
399 ** Set *ppData to point to the Fts5Bm25Data object for the current query. | 544 ** Set *ppData to point to the Fts5Bm25Data object for the current query. |
400 ** If the object has not already been allocated, allocate and populate it | 545 ** If the object has not already been allocated, allocate and populate it |
401 ** now. | 546 ** now. |
402 */ | 547 */ |
403 static int fts5Bm25GetData( | 548 static int fts5Bm25GetData( |
(...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
537 fts5_extension_function xFunc;/* Callback function */ | 682 fts5_extension_function xFunc;/* Callback function */ |
538 void (*xDestroy)(void*); /* Destructor function */ | 683 void (*xDestroy)(void*); /* Destructor function */ |
539 } aBuiltin [] = { | 684 } aBuiltin [] = { |
540 { "snippet", 0, fts5SnippetFunction, 0 }, | 685 { "snippet", 0, fts5SnippetFunction, 0 }, |
541 { "highlight", 0, fts5HighlightFunction, 0 }, | 686 { "highlight", 0, fts5HighlightFunction, 0 }, |
542 { "bm25", 0, fts5Bm25Function, 0 }, | 687 { "bm25", 0, fts5Bm25Function, 0 }, |
543 }; | 688 }; |
544 int rc = SQLITE_OK; /* Return code */ | 689 int rc = SQLITE_OK; /* Return code */ |
545 int i; /* To iterate through builtin functions */ | 690 int i; /* To iterate through builtin functions */ |
546 | 691 |
547 for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){ | 692 for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){ |
548 rc = pApi->xCreateFunction(pApi, | 693 rc = pApi->xCreateFunction(pApi, |
549 aBuiltin[i].zFunc, | 694 aBuiltin[i].zFunc, |
550 aBuiltin[i].pUserData, | 695 aBuiltin[i].pUserData, |
551 aBuiltin[i].xFunc, | 696 aBuiltin[i].xFunc, |
552 aBuiltin[i].xDestroy | 697 aBuiltin[i].xDestroy |
553 ); | 698 ); |
554 } | 699 } |
555 | 700 |
556 return rc; | 701 return rc; |
557 } | 702 } |
558 | 703 |
559 | 704 |
OLD | NEW |