[sql] Import SQLite 3.17.0.

third_party/sqlite/src/ext/fts5/fts5Int.h

 /*
 ** 2014 May 31
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 **
 ******************************************************************************
 **
 */
 #ifndef _FTS5INT_H
 #define _FTS5INT_H
 
 #include "fts5.h"
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
 
 #include <string.h>
 #include <assert.h>
 
 #ifndef SQLITE_AMALGAMATION
 
 typedef unsigned char  u8;
 typedef unsigned int   u32;
 typedef unsigned short u16;
+typedef short i16;
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
 
-#define ArraySize(x) (sizeof(x) / sizeof(x[0]))
+#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
 
 #define testcase(x)
 #define ALWAYS(x) 1
 #define NEVER(x) 0
 
 #define MIN(x,y) (((x) < (y)) ? (x) : (y))
 #define MAX(x,y) (((x) > (y)) ? (x) : (y))
 
 /*
 ** Constants for the largest and smallest possible 64-bit signed integers.
 */
 # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
 # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
 
 #endif
 
+/* Truncate very long tokens to this many bytes. Hard limit is 
+** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
+** field that occurs at the start of each leaf page (see fts5_index.c). */
+#define FTS5_MAX_TOKEN_SIZE 32768
 
 /*
 ** Maximum number of prefix indexes on single FTS5 table. This must be
 ** less than 32. If it is set to anything large than that, an #error
 ** directive in fts5_index.c will cause the build to fail.
 */
 #define FTS5_MAX_PREFIX_INDEXES 31
 
 #define FTS5_DEFAULT_NEARDIST 10
 #define FTS5_DEFAULT_RANK     "bm25"
@@ skipping 14 matching lines @@
 ** is used for assert() conditions that are true only if it can be 
 ** guranteed that the database is not corrupt.
 */
 #ifdef SQLITE_DEBUG
 extern int sqlite3_fts5_may_be_corrupt;
 # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
 #else
 # define assert_nc(x) assert(x)
 #endif
 
+/* Mark a function parameter as unused, to suppress nuisance compiler
+** warnings. */
+#ifndef UNUSED_PARAM
+# define UNUSED_PARAM(X)  (void)(X)
+#endif
+
+#ifndef UNUSED_PARAM2
+# define UNUSED_PARAM2(X, Y)  (void)(X), (void)(Y)
+#endif
+
 typedef struct Fts5Global Fts5Global;
 typedef struct Fts5Colset Fts5Colset;
 
 /* If a NEAR() clump or phrase may only match a specific set of columns, 
 ** then an object of the following type is used to record the set of columns.
 ** Each entry in the aiCol[] array is a column that may be matched.
 **
 ** This object is used by fts5_expr.c and fts5_index.c.
 */
 struct Fts5Colset {
@@ skipping 51 matching lines @@
   char *zName;                    /* Name of FTS index */
   int nCol;                       /* Number of columns */
   char **azCol;                   /* Column names */
   u8 *abUnindexed;                /* True for unindexed columns */
   int nPrefix;                    /* Number of prefix indexes */
   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
   int eContent;                   /* An FTS5_CONTENT value */
   char *zContent;                 /* content table */ 
   char *zContentRowid;            /* "content_rowid=" option value */ 
   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
+  int eDetail;                    /* FTS5_DETAIL_XXX value */
   char *zContentExprlist;
   Fts5Tokenizer *pTok;
   fts5_tokenizer *pTokApi;
 
   /* Values loaded from the %_config table */
   int iCookie;                    /* Incremented when %_config is modified */
   int pgsz;                       /* Approximate page size used in %_data */
   int nAutomerge;                 /* 'automerge' setting */
   int nCrisisMerge;               /* Maximum allowed segments per level */
+  int nUsermerge;                 /* 'usermerge' setting */
   int nHashSize;                  /* Bytes of memory for in-memory hash */
   char *zRank;                    /* Name of rank function */
   char *zRankArgs;                /* Arguments to rank function */
 
   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   char **pzErrmsg;
 
 #ifdef SQLITE_DEBUG
   int bPrefixIndex;               /* True to use prefix-indexes */
 #endif
 };
 
 /* Current expected value of %_config table 'version' field */
 #define FTS5_CURRENT_VERSION 4
 
 #define FTS5_CONTENT_NORMAL   0
 #define FTS5_CONTENT_NONE     1
 #define FTS5_CONTENT_EXTERNAL 2
 
+#define FTS5_DETAIL_FULL    0
+#define FTS5_DETAIL_NONE    1
+#define FTS5_DETAIL_COLUMNS 2
 
 
 
 int sqlite3Fts5ConfigParse(
     Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
 );
 void sqlite3Fts5ConfigFree(Fts5Config*);
 
 int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
 
@@ skipping 26 matching lines @@
 /*
 ** Buffer object for the incremental building of string data.
 */
 typedef struct Fts5Buffer Fts5Buffer;
 struct Fts5Buffer {
   u8 *p;
   int n;
   int nSpace;
 };
 
-int sqlite3Fts5BufferSize(int*, Fts5Buffer*, int);
+int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
 void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
-void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
+void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
 void sqlite3Fts5BufferFree(Fts5Buffer*);
 void sqlite3Fts5BufferZero(Fts5Buffer*);
 void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
 
 char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
 
 #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
 #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
 #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
 #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
 #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
 
 #define fts5BufferGrow(pRc,pBuf,nn) ( \
-  (pBuf)->n + (nn) <= (pBuf)->nSpace ? 0 : \
+  (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
     sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
 )
 
 /* Write and decode big-endian 32-bit integer values */
 void sqlite3Fts5Put32(u8*, int);
 int sqlite3Fts5Get32(const u8*);
 
 #define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
 #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)
 
@@ skipping 14 matching lines @@
   const u8 *a, int n,             /* Poslist buffer to iterate through */
   Fts5PoslistReader *pIter        /* Iterator object to initialize */
 );
 int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
 
 typedef struct Fts5PoslistWriter Fts5PoslistWriter;
 struct Fts5PoslistWriter {
   i64 iPrev;
 };
 int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
+void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
 
 int sqlite3Fts5PoslistNext64(
   const u8 *a, int n,             /* Buffer containing poslist */
   int *pi,                        /* IN/OUT: Offset within a[] */
   i64 *piOff                      /* IN/OUT: Current offset */
 );
 
 /* Malloc utility */
 void *sqlite3Fts5MallocZero(int *pRc, int nByte);
 char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
 
 /* Character set tests (like isspace(), isalpha() etc.) */
 int sqlite3Fts5IsBareword(char t);
 
+
+/* Bucket of terms object used by the integrity-check in offsets=0 mode. */
+typedef struct Fts5Termset Fts5Termset;
+int sqlite3Fts5TermsetNew(Fts5Termset**);
+int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
+void sqlite3Fts5TermsetFree(Fts5Termset*);
+
 /*
 ** End of interface to code in fts5_buffer.c.
 **************************************************************************/
 
 /**************************************************************************
 ** Interface to code in fts5_index.c. fts5_index.c contains contains code
 ** to access the data stored in the %_data table.
 */
 
 typedef struct Fts5Index Fts5Index;
 typedef struct Fts5IndexIter Fts5IndexIter;
 
+struct Fts5IndexIter {
+  i64 iRowid;
+  const u8 *pData;
+  int nData;
+  u8 bEof;
+};
+
+#define sqlite3Fts5IterEof(x) ((x)->bEof)
+
 /*
 ** Values used as part of the flags argument passed to IndexQuery().
 */
 #define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
 #define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
 #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
 #define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
 
+/* The following are used internally by the fts5_index.c module. They are
+** defined here only to make it easier to avoid clashes with the flags
+** above. */
+#define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
+#define FTS5INDEX_QUERY_NOOUTPUT   0x0020
+
 /*
 ** Create/destroy an Fts5Index object.
 */
 int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
 int sqlite3Fts5IndexClose(Fts5Index *p);
 
 /*
-** for(
-**   sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter);
-**   0==sqlite3Fts5IterEof(pIter);
-**   sqlite3Fts5IterNext(pIter)
-** ){
-**   i64 iRowid = sqlite3Fts5IterRowid(pIter);
-** }
+** Return a simple checksum value based on the arguments.
 */
+u64 sqlite3Fts5IndexEntryCksum(
+  i64 iRowid, 
+  int iCol, 
+  int iPos, 
+  int iIdx,
+  const char *pTerm,
+  int nTerm
+);
+
+/*
+** Argument p points to a buffer containing utf-8 text that is n bytes in 
+** size. Return the number of bytes in the nChar character prefix of the
+** buffer, or 0 if there are less than nChar characters in total.
+*/
+int sqlite3Fts5IndexCharlenToBytelen(
+  const char *p, 
+  int nByte, 
+  int nChar
+);
 
 /*
 ** Open a new iterator to iterate though all rowids that match the 
 ** specified token or token prefix.
 */
 int sqlite3Fts5IndexQuery(
   Fts5Index *p,                   /* FTS index to query */
   const char *pToken, int nToken, /* Token (or prefix) to query for */
   int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
   Fts5Colset *pColset,            /* Match these columns only */
   Fts5IndexIter **ppIter          /* OUT: New iterator object */
 );
 
 /*
 ** The various operations on open token or token prefix iterators opened
 ** using sqlite3Fts5IndexQuery().
 */
-int sqlite3Fts5IterEof(Fts5IndexIter*);
 int sqlite3Fts5IterNext(Fts5IndexIter*);
 int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
-i64 sqlite3Fts5IterRowid(Fts5IndexIter*);
-int sqlite3Fts5IterPoslist(Fts5IndexIter*,Fts5Colset*, const u8**, int*, i64*);
-int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf);
 
 /*
 ** Close an iterator opened by sqlite3Fts5IndexQuery().
 */
 void sqlite3Fts5IterClose(Fts5IndexIter*);
 
 /*
 ** This interface is used by the fts5vocab module.
 */
 const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
@@ skipping 43 matching lines @@
 
 /*
 ** Get or set the "averages" values.
 */
 int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
 int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
 
 /*
 ** Functions called by the storage module as part of integrity-check.
 */
-u64 sqlite3Fts5IndexCksum(Fts5Config*,i64,int,int,const char*,int);
 int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
 
 /* 
 ** Called during virtual module initialization to register UDF 
 ** fts5_decode() with SQLite 
 */
 int sqlite3Fts5IndexInit(sqlite3*);
 
 int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
 
 /*
 ** Return the total number of entries read from the %_data table by 
 ** this connection since it was created.
 */
 int sqlite3Fts5IndexReads(Fts5Index *p);
 
 int sqlite3Fts5IndexReinit(Fts5Index *p);
 int sqlite3Fts5IndexOptimize(Fts5Index *p);
 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
+int sqlite3Fts5IndexReset(Fts5Index *p);
 
 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
 
 /**************************************************************************
 ** Interface to code in fts5_varint.c. 
 */
@@ skipping 39 matching lines @@
 **************************************************************************/
 
 /**************************************************************************
 ** Interface to code in fts5_hash.c. 
 */
 typedef struct Fts5Hash Fts5Hash;
 
 /*
 ** Create a hash table, free a hash table.
 */
-int sqlite3Fts5HashNew(Fts5Hash**, int *pnSize);
+int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
 void sqlite3Fts5HashFree(Fts5Hash*);
 
 int sqlite3Fts5HashWrite(
   Fts5Hash*,
   i64 iRowid,                     /* Rowid for this entry */
   int iCol,                       /* Column token appears in (-ve -> delete) */
   int iPos,                       /* Position of token within column */
   char bByte,
   const char *pToken, int nToken  /* Token to add or remove to or from index */
 );
@@ skipping 38 matching lines @@
 
 typedef struct Fts5Storage Fts5Storage;
 
 int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
 int sqlite3Fts5StorageClose(Fts5Storage *p);
 int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 
 int sqlite3Fts5DropAll(Fts5Config*);
 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
 
-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
 
 int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
 
 int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
 void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
 
 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
 
 int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
 int sqlite3Fts5StorageRollback(Fts5Storage *p);
 
 int sqlite3Fts5StorageConfigValue(
     Fts5Storage *p, const char*, sqlite3_value*, int
 );
 
-int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
-
 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
+int sqlite3Fts5StorageReset(Fts5Storage *p);
 
 /*
 ** End of interface to code in fts5_storage.c.
 **************************************************************************/
 
 
 /**************************************************************************
 ** Interface to code in fts5_expr.c. 
 */
 typedef struct Fts5Expr Fts5Expr;
@@ skipping 32 matching lines @@
 
 void sqlite3Fts5ExprFree(Fts5Expr*);
 
 /* Called during startup to register a UDF with SQLite */
 int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
 
 int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
 
-int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
+typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
+Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
+int sqlite3Fts5ExprPopulatePoslists(
+    Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
+);
+void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
+
+int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
+
+int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
 
 /*******************************************
 ** The fts5_expr.c API above this point is used by the other hand-written
 ** C code in this module. The interfaces below this point are called by
 ** the parser code in fts5parse.y.  */
 
 void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
 
 Fts5ExprNode *sqlite3Fts5ParseNode(
   Fts5Parse *pParse,
   int eType,
   Fts5ExprNode *pLeft,
   Fts5ExprNode *pRight,
   Fts5ExprNearset *pNear
 );
 
+Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
+  Fts5Parse *pParse,
+  Fts5ExprNode *pLeft,
+  Fts5ExprNode *pRight
+);
+
 Fts5ExprPhrase *sqlite3Fts5ParseTerm(
   Fts5Parse *pParse, 
   Fts5ExprPhrase *pPhrase, 
   Fts5Token *pToken,
   int bPrefix
 );
 
 Fts5ExprNearset *sqlite3Fts5ParseNearset(
   Fts5Parse*, 
   Fts5ExprNearset*,
   Fts5ExprPhrase* 
 );
 
 Fts5Colset *sqlite3Fts5ParseColset(
   Fts5Parse*, 
   Fts5Colset*, 
   Fts5Token *
 );
 
 void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
 void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
 
 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
 void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
+Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
 
 /*
 ** End of interface to code in fts5_expr.c.
 **************************************************************************/
 
 
 
 /**************************************************************************
@@ skipping 29 matching lines @@
 ** Interface to automatically generated code in fts5_unicode2.c. 
 */
 int sqlite3Fts5UnicodeIsalnum(int c);
 int sqlite3Fts5UnicodeIsdiacritic(int c);
 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
 /*
 ** End of interface to code in fts5_unicode2.c.
 **************************************************************************/
 
 #endif