| Index: third_party/sqlite/amalgamation/sqlite3.07.c
|
| diff --git a/third_party/sqlite/amalgamation/sqlite3.07.c b/third_party/sqlite/amalgamation/sqlite3.07.c
|
| new file mode 100644
|
| index 0000000000000000000000000000000000000000..452bc768526e1e2d6d501c0fb01daa3d241fc7d2
|
| --- /dev/null
|
| +++ b/third_party/sqlite/amalgamation/sqlite3.07.c
|
| @@ -0,0 +1,26322 @@
|
| +/************** Begin file sqlite3rbu.c **************************************/
|
| +/*
|
| +** 2014 August 30
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +*************************************************************************
|
| +**
|
| +**
|
| +** OVERVIEW
|
| +**
|
| +** The RBU extension requires that the RBU update be packaged as an
|
| +** SQLite database. The tables it expects to find are described in
|
| +** sqlite3rbu.h. Essentially, for each table xyz in the target database
|
| +** that the user wishes to write to, a corresponding data_xyz table is
|
| +** created in the RBU database and populated with one row for each row to
|
| +** update, insert or delete from the target table.
|
| +**
|
| +** The update proceeds in three stages:
|
| +**
|
| +** 1) The database is updated. The modified database pages are written
|
| +** to a *-oal file. A *-oal file is just like a *-wal file, except
|
| +** that it is named "<database>-oal" instead of "<database>-wal".
|
| +** Because regular SQLite clients do not look for file named
|
| +** "<database>-oal", they go on using the original database in
|
| +** rollback mode while the *-oal file is being generated.
|
| +**
|
| +** During this stage RBU does not update the database by writing
|
| +** directly to the target tables. Instead it creates "imposter"
|
| +** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses
|
| +** to update each b-tree individually. All updates required by each
|
| +** b-tree are completed before moving on to the next, and all
|
| +** updates are done in sorted key order.
|
| +**
|
| +** 2) The "<database>-oal" file is moved to the equivalent "<database>-wal"
|
| +** location using a call to rename(2). Before doing this the RBU
|
| +** module takes an EXCLUSIVE lock on the database file, ensuring
|
| +** that there are no other active readers.
|
| +**
|
| +** Once the EXCLUSIVE lock is released, any other database readers
|
| +** detect the new *-wal file and read the database in wal mode. At
|
| +** this point they see the new version of the database - including
|
| +** the updates made as part of the RBU update.
|
| +**
|
| +** 3) The new *-wal file is checkpointed. This proceeds in the same way
|
| +** as a regular database checkpoint, except that a single frame is
|
| +** checkpointed each time sqlite3rbu_step() is called. If the RBU
|
| +** handle is closed before the entire *-wal file is checkpointed,
|
| +** the checkpoint progress is saved in the RBU database and the
|
| +** checkpoint can be resumed by another RBU client at some point in
|
| +** the future.
|
| +**
|
| +** POTENTIAL PROBLEMS
|
| +**
|
| +** The rename() call might not be portable. And RBU is not currently
|
| +** syncing the directory after renaming the file.
|
| +**
|
| +** When state is saved, any commit to the *-oal file and the commit to
|
| +** the RBU update database are not atomic. So if the power fails at the
|
| +** wrong moment they might get out of sync. As the main database will be
|
| +** committed before the RBU update database this will likely either just
|
| +** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE
|
| +** constraint violations).
|
| +**
|
| +** If some client does modify the target database mid RBU update, or some
|
| +** other error occurs, the RBU extension will keep throwing errors. It's
|
| +** not really clear how to get out of this state. The system could just
|
| +** by delete the RBU update database and *-oal file and have the device
|
| +** download the update again and start over.
|
| +**
|
| +** At present, for an UPDATE, both the new.* and old.* records are
|
| +** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all
|
| +** fields are collected. This means we're probably writing a lot more
|
| +** data to disk when saving the state of an ongoing update to the RBU
|
| +** update database than is strictly necessary.
|
| +**
|
| +*/
|
| +
|
| +/* #include <assert.h> */
|
| +/* #include <string.h> */
|
| +/* #include <stdio.h> */
|
| +
|
| +/* #include "sqlite3.h" */
|
| +
|
| +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU)
|
| +/************** Include sqlite3rbu.h in the middle of sqlite3rbu.c ***********/
|
| +/************** Begin file sqlite3rbu.h **************************************/
|
| +/*
|
| +** 2014 August 30
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +*************************************************************************
|
| +**
|
| +** This file contains the public interface for the RBU extension.
|
| +*/
|
| +
|
| +/*
|
| +** SUMMARY
|
| +**
|
| +** Writing a transaction containing a large number of operations on
|
| +** b-tree indexes that are collectively larger than the available cache
|
| +** memory can be very inefficient.
|
| +**
|
| +** The problem is that in order to update a b-tree, the leaf page (at least)
|
| +** containing the entry being inserted or deleted must be modified. If the
|
| +** working set of leaves is larger than the available cache memory, then a
|
| +** single leaf that is modified more than once as part of the transaction
|
| +** may be loaded from or written to the persistent media multiple times.
|
| +** Additionally, because the index updates are likely to be applied in
|
| +** random order, access to pages within the database is also likely to be in
|
| +** random order, which is itself quite inefficient.
|
| +**
|
| +** One way to improve the situation is to sort the operations on each index
|
| +** by index key before applying them to the b-tree. This leads to an IO
|
| +** pattern that resembles a single linear scan through the index b-tree,
|
| +** and all but guarantees each modified leaf page is loaded and stored
|
| +** exactly once. SQLite uses this trick to improve the performance of
|
| +** CREATE INDEX commands. This extension allows it to be used to improve
|
| +** the performance of large transactions on existing databases.
|
| +**
|
| +** Additionally, this extension allows the work involved in writing the
|
| +** large transaction to be broken down into sub-transactions performed
|
| +** sequentially by separate processes. This is useful if the system cannot
|
| +** guarantee that a single update process will run for long enough to apply
|
| +** the entire update, for example because the update is being applied on a
|
| +** mobile device that is frequently rebooted. Even after the writer process
|
| +** has committed one or more sub-transactions, other database clients continue
|
| +** to read from the original database snapshot. In other words, partially
|
| +** applied transactions are not visible to other clients.
|
| +**
|
| +** "RBU" stands for "Resumable Bulk Update". As in a large database update
|
| +** transmitted via a wireless network to a mobile device. A transaction
|
| +** applied using this extension is hence refered to as an "RBU update".
|
| +**
|
| +**
|
| +** LIMITATIONS
|
| +**
|
| +** An "RBU update" transaction is subject to the following limitations:
|
| +**
|
| +** * The transaction must consist of INSERT, UPDATE and DELETE operations
|
| +** only.
|
| +**
|
| +** * INSERT statements may not use any default values.
|
| +**
|
| +** * UPDATE and DELETE statements must identify their target rows by
|
| +** non-NULL PRIMARY KEY values. Rows with NULL values stored in PRIMARY
|
| +** KEY fields may not be updated or deleted. If the table being written
|
| +** has no PRIMARY KEY, affected rows must be identified by rowid.
|
| +**
|
| +** * UPDATE statements may not modify PRIMARY KEY columns.
|
| +**
|
| +** * No triggers will be fired.
|
| +**
|
| +** * No foreign key violations are detected or reported.
|
| +**
|
| +** * CHECK constraints are not enforced.
|
| +**
|
| +** * No constraint handling mode except for "OR ROLLBACK" is supported.
|
| +**
|
| +**
|
| +** PREPARATION
|
| +**
|
| +** An "RBU update" is stored as a separate SQLite database. A database
|
| +** containing an RBU update is an "RBU database". For each table in the
|
| +** target database to be updated, the RBU database should contain a table
|
| +** named "data_<target name>" containing the same set of columns as the
|
| +** target table, and one more - "rbu_control". The data_% table should
|
| +** have no PRIMARY KEY or UNIQUE constraints, but each column should have
|
| +** the same type as the corresponding column in the target database.
|
| +** The "rbu_control" column should have no type at all. For example, if
|
| +** the target database contains:
|
| +**
|
| +** CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c UNIQUE);
|
| +**
|
| +** Then the RBU database should contain:
|
| +**
|
| +** CREATE TABLE data_t1(a INTEGER, b TEXT, c, rbu_control);
|
| +**
|
| +** The order of the columns in the data_% table does not matter.
|
| +**
|
| +** Instead of a regular table, the RBU database may also contain virtual
|
| +** tables or view named using the data_<target> naming scheme.
|
| +**
|
| +** Instead of the plain data_<target> naming scheme, RBU database tables
|
| +** may also be named data<integer>_<target>, where <integer> is any sequence
|
| +** of zero or more numeric characters (0-9). This can be significant because
|
| +** tables within the RBU database are always processed in order sorted by
|
| +** name. By judicious selection of the the <integer> portion of the names
|
| +** of the RBU tables the user can therefore control the order in which they
|
| +** are processed. This can be useful, for example, to ensure that "external
|
| +** content" FTS4 tables are updated before their underlying content tables.
|
| +**
|
| +** If the target database table is a virtual table or a table that has no
|
| +** PRIMARY KEY declaration, the data_% table must also contain a column
|
| +** named "rbu_rowid". This column is mapped to the tables implicit primary
|
| +** key column - "rowid". Virtual tables for which the "rowid" column does
|
| +** not function like a primary key value cannot be updated using RBU. For
|
| +** example, if the target db contains either of the following:
|
| +**
|
| +** CREATE VIRTUAL TABLE x1 USING fts3(a, b);
|
| +** CREATE TABLE x1(a, b)
|
| +**
|
| +** then the RBU database should contain:
|
| +**
|
| +** CREATE TABLE data_x1(a, b, rbu_rowid, rbu_control);
|
| +**
|
| +** All non-hidden columns (i.e. all columns matched by "SELECT *") of the
|
| +** target table must be present in the input table. For virtual tables,
|
| +** hidden columns are optional - they are updated by RBU if present in
|
| +** the input table, or not otherwise. For example, to write to an fts4
|
| +** table with a hidden languageid column such as:
|
| +**
|
| +** CREATE VIRTUAL TABLE ft1 USING fts4(a, b, languageid='langid');
|
| +**
|
| +** Either of the following input table schemas may be used:
|
| +**
|
| +** CREATE TABLE data_ft1(a, b, langid, rbu_rowid, rbu_control);
|
| +** CREATE TABLE data_ft1(a, b, rbu_rowid, rbu_control);
|
| +**
|
| +** For each row to INSERT into the target database as part of the RBU
|
| +** update, the corresponding data_% table should contain a single record
|
| +** with the "rbu_control" column set to contain integer value 0. The
|
| +** other columns should be set to the values that make up the new record
|
| +** to insert.
|
| +**
|
| +** If the target database table has an INTEGER PRIMARY KEY, it is not
|
| +** possible to insert a NULL value into the IPK column. Attempting to
|
| +** do so results in an SQLITE_MISMATCH error.
|
| +**
|
| +** For each row to DELETE from the target database as part of the RBU
|
| +** update, the corresponding data_% table should contain a single record
|
| +** with the "rbu_control" column set to contain integer value 1. The
|
| +** real primary key values of the row to delete should be stored in the
|
| +** corresponding columns of the data_% table. The values stored in the
|
| +** other columns are not used.
|
| +**
|
| +** For each row to UPDATE from the target database as part of the RBU
|
| +** update, the corresponding data_% table should contain a single record
|
| +** with the "rbu_control" column set to contain a value of type text.
|
| +** The real primary key values identifying the row to update should be
|
| +** stored in the corresponding columns of the data_% table row, as should
|
| +** the new values of all columns being update. The text value in the
|
| +** "rbu_control" column must contain the same number of characters as
|
| +** there are columns in the target database table, and must consist entirely
|
| +** of 'x' and '.' characters (or in some special cases 'd' - see below). For
|
| +** each column that is being updated, the corresponding character is set to
|
| +** 'x'. For those that remain as they are, the corresponding character of the
|
| +** rbu_control value should be set to '.'. For example, given the tables
|
| +** above, the update statement:
|
| +**
|
| +** UPDATE t1 SET c = 'usa' WHERE a = 4;
|
| +**
|
| +** is represented by the data_t1 row created by:
|
| +**
|
| +** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..x');
|
| +**
|
| +** Instead of an 'x' character, characters of the rbu_control value specified
|
| +** for UPDATEs may also be set to 'd'. In this case, instead of updating the
|
| +** target table with the value stored in the corresponding data_% column, the
|
| +** user-defined SQL function "rbu_delta()" is invoked and the result stored in
|
| +** the target table column. rbu_delta() is invoked with two arguments - the
|
| +** original value currently stored in the target table column and the
|
| +** value specified in the data_xxx table.
|
| +**
|
| +** For example, this row:
|
| +**
|
| +** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..d');
|
| +**
|
| +** is similar to an UPDATE statement such as:
|
| +**
|
| +** UPDATE t1 SET c = rbu_delta(c, 'usa') WHERE a = 4;
|
| +**
|
| +** Finally, if an 'f' character appears in place of a 'd' or 's' in an
|
| +** ota_control string, the contents of the data_xxx table column is assumed
|
| +** to be a "fossil delta" - a patch to be applied to a blob value in the
|
| +** format used by the fossil source-code management system. In this case
|
| +** the existing value within the target database table must be of type BLOB.
|
| +** It is replaced by the result of applying the specified fossil delta to
|
| +** itself.
|
| +**
|
| +** If the target database table is a virtual table or a table with no PRIMARY
|
| +** KEY, the rbu_control value should not include a character corresponding
|
| +** to the rbu_rowid value. For example, this:
|
| +**
|
| +** INSERT INTO data_ft1(a, b, rbu_rowid, rbu_control)
|
| +** VALUES(NULL, 'usa', 12, '.x');
|
| +**
|
| +** causes a result similar to:
|
| +**
|
| +** UPDATE ft1 SET b = 'usa' WHERE rowid = 12;
|
| +**
|
| +** The data_xxx tables themselves should have no PRIMARY KEY declarations.
|
| +** However, RBU is more efficient if reading the rows in from each data_xxx
|
| +** table in "rowid" order is roughly the same as reading them sorted by
|
| +** the PRIMARY KEY of the corresponding target database table. In other
|
| +** words, rows should be sorted using the destination table PRIMARY KEY
|
| +** fields before they are inserted into the data_xxx tables.
|
| +**
|
| +** USAGE
|
| +**
|
| +** The API declared below allows an application to apply an RBU update
|
| +** stored on disk to an existing target database. Essentially, the
|
| +** application:
|
| +**
|
| +** 1) Opens an RBU handle using the sqlite3rbu_open() function.
|
| +**
|
| +** 2) Registers any required virtual table modules with the database
|
| +** handle returned by sqlite3rbu_db(). Also, if required, register
|
| +** the rbu_delta() implementation.
|
| +**
|
| +** 3) Calls the sqlite3rbu_step() function one or more times on
|
| +** the new handle. Each call to sqlite3rbu_step() performs a single
|
| +** b-tree operation, so thousands of calls may be required to apply
|
| +** a complete update.
|
| +**
|
| +** 4) Calls sqlite3rbu_close() to close the RBU update handle. If
|
| +** sqlite3rbu_step() has been called enough times to completely
|
| +** apply the update to the target database, then the RBU database
|
| +** is marked as fully applied. Otherwise, the state of the RBU
|
| +** update application is saved in the RBU database for later
|
| +** resumption.
|
| +**
|
| +** See comments below for more detail on APIs.
|
| +**
|
| +** If an update is only partially applied to the target database by the
|
| +** time sqlite3rbu_close() is called, various state information is saved
|
| +** within the RBU database. This allows subsequent processes to automatically
|
| +** resume the RBU update from where it left off.
|
| +**
|
| +** To remove all RBU extension state information, returning an RBU database
|
| +** to its original contents, it is sufficient to drop all tables that begin
|
| +** with the prefix "rbu_"
|
| +**
|
| +** DATABASE LOCKING
|
| +**
|
| +** An RBU update may not be applied to a database in WAL mode. Attempting
|
| +** to do so is an error (SQLITE_ERROR).
|
| +**
|
| +** While an RBU handle is open, a SHARED lock may be held on the target
|
| +** database file. This means it is possible for other clients to read the
|
| +** database, but not to write it.
|
| +**
|
| +** If an RBU update is started and then suspended before it is completed,
|
| +** then an external client writes to the database, then attempting to resume
|
| +** the suspended RBU update is also an error (SQLITE_BUSY).
|
| +*/
|
| +
|
| +#ifndef _SQLITE3RBU_H
|
| +#define _SQLITE3RBU_H
|
| +
|
| +/* #include "sqlite3.h" ** Required for error code definitions ** */
|
| +
|
| +#if 0
|
| +extern "C" {
|
| +#endif
|
| +
|
| +typedef struct sqlite3rbu sqlite3rbu;
|
| +
|
| +/*
|
| +** Open an RBU handle.
|
| +**
|
| +** Argument zTarget is the path to the target database. Argument zRbu is
|
| +** the path to the RBU database. Each call to this function must be matched
|
| +** by a call to sqlite3rbu_close(). When opening the databases, RBU passes
|
| +** the SQLITE_CONFIG_URI flag to sqlite3_open_v2(). So if either zTarget
|
| +** or zRbu begin with "file:", it will be interpreted as an SQLite
|
| +** database URI, not a regular file name.
|
| +**
|
| +** If the zState argument is passed a NULL value, the RBU extension stores
|
| +** the current state of the update (how many rows have been updated, which
|
| +** indexes are yet to be updated etc.) within the RBU database itself. This
|
| +** can be convenient, as it means that the RBU application does not need to
|
| +** organize removing a separate state file after the update is concluded.
|
| +** Or, if zState is non-NULL, it must be a path to a database file in which
|
| +** the RBU extension can store the state of the update.
|
| +**
|
| +** When resuming an RBU update, the zState argument must be passed the same
|
| +** value as when the RBU update was started.
|
| +**
|
| +** Once the RBU update is finished, the RBU extension does not
|
| +** automatically remove any zState database file, even if it created it.
|
| +**
|
| +** By default, RBU uses the default VFS to access the files on disk. To
|
| +** use a VFS other than the default, an SQLite "file:" URI containing a
|
| +** "vfs=..." option may be passed as the zTarget option.
|
| +**
|
| +** IMPORTANT NOTE FOR ZIPVFS USERS: The RBU extension works with all of
|
| +** SQLite's built-in VFSs, including the multiplexor VFS. However it does
|
| +** not work out of the box with zipvfs. Refer to the comment describing
|
| +** the zipvfs_create_vfs() API below for details on using RBU with zipvfs.
|
| +*/
|
| +SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open(
|
| + const char *zTarget,
|
| + const char *zRbu,
|
| + const char *zState
|
| +);
|
| +
|
| +/*
|
| +** Internally, each RBU connection uses a separate SQLite database
|
| +** connection to access the target and rbu update databases. This
|
| +** API allows the application direct access to these database handles.
|
| +**
|
| +** The first argument passed to this function must be a valid, open, RBU
|
| +** handle. The second argument should be passed zero to access the target
|
| +** database handle, or non-zero to access the rbu update database handle.
|
| +** Accessing the underlying database handles may be useful in the
|
| +** following scenarios:
|
| +**
|
| +** * If any target tables are virtual tables, it may be necessary to
|
| +** call sqlite3_create_module() on the target database handle to
|
| +** register the required virtual table implementations.
|
| +**
|
| +** * If the data_xxx tables in the RBU source database are virtual
|
| +** tables, the application may need to call sqlite3_create_module() on
|
| +** the rbu update db handle to any required virtual table
|
| +** implementations.
|
| +**
|
| +** * If the application uses the "rbu_delta()" feature described above,
|
| +** it must use sqlite3_create_function() or similar to register the
|
| +** rbu_delta() implementation with the target database handle.
|
| +**
|
| +** If an error has occurred, either while opening or stepping the RBU object,
|
| +** this function may return NULL. The error code and message may be collected
|
| +** when sqlite3rbu_close() is called.
|
| +**
|
| +** Database handles returned by this function remain valid until the next
|
| +** call to any sqlite3rbu_xxx() function other than sqlite3rbu_db().
|
| +*/
|
| +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3rbu_db(sqlite3rbu*, int bRbu);
|
| +
|
| +/*
|
| +** Do some work towards applying the RBU update to the target db.
|
| +**
|
| +** Return SQLITE_DONE if the update has been completely applied, or
|
| +** SQLITE_OK if no error occurs but there remains work to do to apply
|
| +** the RBU update. If an error does occur, some other error code is
|
| +** returned.
|
| +**
|
| +** Once a call to sqlite3rbu_step() has returned a value other than
|
| +** SQLITE_OK, all subsequent calls on the same RBU handle are no-ops
|
| +** that immediately return the same value.
|
| +*/
|
| +SQLITE_API int SQLITE_STDCALL sqlite3rbu_step(sqlite3rbu *pRbu);
|
| +
|
| +/*
|
| +** Force RBU to save its state to disk.
|
| +**
|
| +** If a power failure or application crash occurs during an update, following
|
| +** system recovery RBU may resume the update from the point at which the state
|
| +** was last saved. In other words, from the most recent successful call to
|
| +** sqlite3rbu_close() or this function.
|
| +**
|
| +** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
|
| +*/
|
| +SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *pRbu);
|
| +
|
| +/*
|
| +** Close an RBU handle.
|
| +**
|
| +** If the RBU update has been completely applied, mark the RBU database
|
| +** as fully applied. Otherwise, assuming no error has occurred, save the
|
| +** current state of the RBU update appliation to the RBU database.
|
| +**
|
| +** If an error has already occurred as part of an sqlite3rbu_step()
|
| +** or sqlite3rbu_open() call, or if one occurs within this function, an
|
| +** SQLite error code is returned. Additionally, *pzErrmsg may be set to
|
| +** point to a buffer containing a utf-8 formatted English language error
|
| +** message. It is the responsibility of the caller to eventually free any
|
| +** such buffer using sqlite3_free().
|
| +**
|
| +** Otherwise, if no error occurs, this function returns SQLITE_OK if the
|
| +** update has been partially applied, or SQLITE_DONE if it has been
|
| +** completely applied.
|
| +*/
|
| +SQLITE_API int SQLITE_STDCALL sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg);
|
| +
|
| +/*
|
| +** Return the total number of key-value operations (inserts, deletes or
|
| +** updates) that have been performed on the target database since the
|
| +** current RBU update was started.
|
| +*/
|
| +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3rbu_progress(sqlite3rbu *pRbu);
|
| +
|
| +/*
|
| +** Create an RBU VFS named zName that accesses the underlying file-system
|
| +** via existing VFS zParent. Or, if the zParent parameter is passed NULL,
|
| +** then the new RBU VFS uses the default system VFS to access the file-system.
|
| +** The new object is registered as a non-default VFS with SQLite before
|
| +** returning.
|
| +**
|
| +** Part of the RBU implementation uses a custom VFS object. Usually, this
|
| +** object is created and deleted automatically by RBU.
|
| +**
|
| +** The exception is for applications that also use zipvfs. In this case,
|
| +** the custom VFS must be explicitly created by the user before the RBU
|
| +** handle is opened. The RBU VFS should be installed so that the zipvfs
|
| +** VFS uses the RBU VFS, which in turn uses any other VFS layers in use
|
| +** (for example multiplexor) to access the file-system. For example,
|
| +** to assemble an RBU enabled VFS stack that uses both zipvfs and
|
| +** multiplexor (error checking omitted):
|
| +**
|
| +** // Create a VFS named "multiplex" (not the default).
|
| +** sqlite3_multiplex_initialize(0, 0);
|
| +**
|
| +** // Create an rbu VFS named "rbu" that uses multiplexor. If the
|
| +** // second argument were replaced with NULL, the "rbu" VFS would
|
| +** // access the file-system via the system default VFS, bypassing the
|
| +** // multiplexor.
|
| +** sqlite3rbu_create_vfs("rbu", "multiplex");
|
| +**
|
| +** // Create a zipvfs VFS named "zipvfs" that uses rbu.
|
| +** zipvfs_create_vfs_v3("zipvfs", "rbu", 0, xCompressorAlgorithmDetector);
|
| +**
|
| +** // Make zipvfs the default VFS.
|
| +** sqlite3_vfs_register(sqlite3_vfs_find("zipvfs"), 1);
|
| +**
|
| +** Because the default VFS created above includes a RBU functionality, it
|
| +** may be used by RBU clients. Attempting to use RBU with a zipvfs VFS stack
|
| +** that does not include the RBU layer results in an error.
|
| +**
|
| +** The overhead of adding the "rbu" VFS to the system is negligible for
|
| +** non-RBU users. There is no harm in an application accessing the
|
| +** file-system via "rbu" all the time, even if it only uses RBU functionality
|
| +** occasionally.
|
| +*/
|
| +SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const char *zParent);
|
| +
|
| +/*
|
| +** Deregister and destroy an RBU vfs created by an earlier call to
|
| +** sqlite3rbu_create_vfs().
|
| +**
|
| +** VFS objects are not reference counted. If a VFS object is destroyed
|
| +** before all database handles that use it have been closed, the results
|
| +** are undefined.
|
| +*/
|
| +SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName);
|
| +
|
| +#if 0
|
| +} /* end of the 'extern "C"' block */
|
| +#endif
|
| +
|
| +#endif /* _SQLITE3RBU_H */
|
| +
|
| +/************** End of sqlite3rbu.h ******************************************/
|
| +/************** Continuing where we left off in sqlite3rbu.c *****************/
|
| +
|
| +#if defined(_WIN32_WCE)
|
| +/* #include "windows.h" */
|
| +#endif
|
| +
|
| +/* Maximum number of prepared UPDATE statements held by this module */
|
| +#define SQLITE_RBU_UPDATE_CACHESIZE 16
|
| +
|
| +/*
|
| +** Swap two objects of type TYPE.
|
| +*/
|
| +#if !defined(SQLITE_AMALGAMATION)
|
| +# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;}
|
| +#endif
|
| +
|
| +/*
|
| +** The rbu_state table is used to save the state of a partially applied
|
| +** update so that it can be resumed later. The table consists of integer
|
| +** keys mapped to values as follows:
|
| +**
|
| +** RBU_STATE_STAGE:
|
| +** May be set to integer values 1, 2, 4 or 5. As follows:
|
| +** 1: the *-rbu file is currently under construction.
|
| +** 2: the *-rbu file has been constructed, but not yet moved
|
| +** to the *-wal path.
|
| +** 4: the checkpoint is underway.
|
| +** 5: the rbu update has been checkpointed.
|
| +**
|
| +** RBU_STATE_TBL:
|
| +** Only valid if STAGE==1. The target database name of the table
|
| +** currently being written.
|
| +**
|
| +** RBU_STATE_IDX:
|
| +** Only valid if STAGE==1. The target database name of the index
|
| +** currently being written, or NULL if the main table is currently being
|
| +** updated.
|
| +**
|
| +** RBU_STATE_ROW:
|
| +** Only valid if STAGE==1. Number of rows already processed for the current
|
| +** table/index.
|
| +**
|
| +** RBU_STATE_PROGRESS:
|
| +** Trbul number of sqlite3rbu_step() calls made so far as part of this
|
| +** rbu update.
|
| +**
|
| +** RBU_STATE_CKPT:
|
| +** Valid if STAGE==4. The 64-bit checksum associated with the wal-index
|
| +** header created by recovering the *-wal file. This is used to detect
|
| +** cases when another client appends frames to the *-wal file in the
|
| +** middle of an incremental checkpoint (an incremental checkpoint cannot
|
| +** be continued if this happens).
|
| +**
|
| +** RBU_STATE_COOKIE:
|
| +** Valid if STAGE==1. The current change-counter cookie value in the
|
| +** target db file.
|
| +**
|
| +** RBU_STATE_OALSZ:
|
| +** Valid if STAGE==1. The size in bytes of the *-oal file.
|
| +*/
|
| +#define RBU_STATE_STAGE 1
|
| +#define RBU_STATE_TBL 2
|
| +#define RBU_STATE_IDX 3
|
| +#define RBU_STATE_ROW 4
|
| +#define RBU_STATE_PROGRESS 5
|
| +#define RBU_STATE_CKPT 6
|
| +#define RBU_STATE_COOKIE 7
|
| +#define RBU_STATE_OALSZ 8
|
| +
|
| +#define RBU_STAGE_OAL 1
|
| +#define RBU_STAGE_MOVE 2
|
| +#define RBU_STAGE_CAPTURE 3
|
| +#define RBU_STAGE_CKPT 4
|
| +#define RBU_STAGE_DONE 5
|
| +
|
| +
|
| +#define RBU_CREATE_STATE \
|
| + "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)"
|
| +
|
| +typedef struct RbuFrame RbuFrame;
|
| +typedef struct RbuObjIter RbuObjIter;
|
| +typedef struct RbuState RbuState;
|
| +typedef struct rbu_vfs rbu_vfs;
|
| +typedef struct rbu_file rbu_file;
|
| +typedef struct RbuUpdateStmt RbuUpdateStmt;
|
| +
|
| +#if !defined(SQLITE_AMALGAMATION)
|
| +typedef unsigned int u32;
|
| +typedef unsigned char u8;
|
| +typedef sqlite3_int64 i64;
|
| +#endif
|
| +
|
| +/*
|
| +** These values must match the values defined in wal.c for the equivalent
|
| +** locks. These are not magic numbers as they are part of the SQLite file
|
| +** format.
|
| +*/
|
| +#define WAL_LOCK_WRITE 0
|
| +#define WAL_LOCK_CKPT 1
|
| +#define WAL_LOCK_READ0 3
|
| +
|
| +/*
|
| +** A structure to store values read from the rbu_state table in memory.
|
| +*/
|
| +struct RbuState {
|
| + int eStage;
|
| + char *zTbl;
|
| + char *zIdx;
|
| + i64 iWalCksum;
|
| + int nRow;
|
| + i64 nProgress;
|
| + u32 iCookie;
|
| + i64 iOalSz;
|
| +};
|
| +
|
| +struct RbuUpdateStmt {
|
| + char *zMask; /* Copy of update mask used with pUpdate */
|
| + sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */
|
| + RbuUpdateStmt *pNext;
|
| +};
|
| +
|
| +/*
|
| +** An iterator of this type is used to iterate through all objects in
|
| +** the target database that require updating. For each such table, the
|
| +** iterator visits, in order:
|
| +**
|
| +** * the table itself,
|
| +** * each index of the table (zero or more points to visit), and
|
| +** * a special "cleanup table" state.
|
| +**
|
| +** abIndexed:
|
| +** If the table has no indexes on it, abIndexed is set to NULL. Otherwise,
|
| +** it points to an array of flags nTblCol elements in size. The flag is
|
| +** set for each column that is either a part of the PK or a part of an
|
| +** index. Or clear otherwise.
|
| +**
|
| +*/
|
| +struct RbuObjIter {
|
| + sqlite3_stmt *pTblIter; /* Iterate through tables */
|
| + sqlite3_stmt *pIdxIter; /* Index iterator */
|
| + int nTblCol; /* Size of azTblCol[] array */
|
| + char **azTblCol; /* Array of unquoted target column names */
|
| + char **azTblType; /* Array of target column types */
|
| + int *aiSrcOrder; /* src table col -> target table col */
|
| + u8 *abTblPk; /* Array of flags, set on target PK columns */
|
| + u8 *abNotNull; /* Array of flags, set on NOT NULL columns */
|
| + u8 *abIndexed; /* Array of flags, set on indexed & PK cols */
|
| + int eType; /* Table type - an RBU_PK_XXX value */
|
| +
|
| + /* Output variables. zTbl==0 implies EOF. */
|
| + int bCleanup; /* True in "cleanup" state */
|
| + const char *zTbl; /* Name of target db table */
|
| + const char *zDataTbl; /* Name of rbu db table (or null) */
|
| + const char *zIdx; /* Name of target db index (or null) */
|
| + int iTnum; /* Root page of current object */
|
| + int iPkTnum; /* If eType==EXTERNAL, root of PK index */
|
| + int bUnique; /* Current index is unique */
|
| +
|
| + /* Statements created by rbuObjIterPrepareAll() */
|
| + int nCol; /* Number of columns in current object */
|
| + sqlite3_stmt *pSelect; /* Source data */
|
| + sqlite3_stmt *pInsert; /* Statement for INSERT operations */
|
| + sqlite3_stmt *pDelete; /* Statement for DELETE ops */
|
| + sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */
|
| +
|
| + /* Last UPDATE used (for PK b-tree updates only), or NULL. */
|
| + RbuUpdateStmt *pRbuUpdate;
|
| +};
|
| +
|
| +/*
|
| +** Values for RbuObjIter.eType
|
| +**
|
| +** 0: Table does not exist (error)
|
| +** 1: Table has an implicit rowid.
|
| +** 2: Table has an explicit IPK column.
|
| +** 3: Table has an external PK index.
|
| +** 4: Table is WITHOUT ROWID.
|
| +** 5: Table is a virtual table.
|
| +*/
|
| +#define RBU_PK_NOTABLE 0
|
| +#define RBU_PK_NONE 1
|
| +#define RBU_PK_IPK 2
|
| +#define RBU_PK_EXTERNAL 3
|
| +#define RBU_PK_WITHOUT_ROWID 4
|
| +#define RBU_PK_VTAB 5
|
| +
|
| +
|
| +/*
|
| +** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs
|
| +** one of the following operations.
|
| +*/
|
| +#define RBU_INSERT 1 /* Insert on a main table b-tree */
|
| +#define RBU_DELETE 2 /* Delete a row from a main table b-tree */
|
| +#define RBU_IDX_DELETE 3 /* Delete a row from an aux. index b-tree */
|
| +#define RBU_IDX_INSERT 4 /* Insert on an aux. index b-tree */
|
| +#define RBU_UPDATE 5 /* Update a row in a main table b-tree */
|
| +
|
| +
|
| +/*
|
| +** A single step of an incremental checkpoint - frame iWalFrame of the wal
|
| +** file should be copied to page iDbPage of the database file.
|
| +*/
|
| +struct RbuFrame {
|
| + u32 iDbPage;
|
| + u32 iWalFrame;
|
| +};
|
| +
|
| +/*
|
| +** RBU handle.
|
| +*/
|
| +struct sqlite3rbu {
|
| + int eStage; /* Value of RBU_STATE_STAGE field */
|
| + sqlite3 *dbMain; /* target database handle */
|
| + sqlite3 *dbRbu; /* rbu database handle */
|
| + char *zTarget; /* Path to target db */
|
| + char *zRbu; /* Path to rbu db */
|
| + char *zState; /* Path to state db (or NULL if zRbu) */
|
| + char zStateDb[5]; /* Db name for state ("stat" or "main") */
|
| + int rc; /* Value returned by last rbu_step() call */
|
| + char *zErrmsg; /* Error message if rc!=SQLITE_OK */
|
| + int nStep; /* Rows processed for current object */
|
| + int nProgress; /* Rows processed for all objects */
|
| + RbuObjIter objiter; /* Iterator for skipping through tbl/idx */
|
| + const char *zVfsName; /* Name of automatically created rbu vfs */
|
| + rbu_file *pTargetFd; /* File handle open on target db */
|
| + i64 iOalSz;
|
| +
|
| + /* The following state variables are used as part of the incremental
|
| + ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding
|
| + ** function rbuSetupCheckpoint() for details. */
|
| + u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */
|
| + u32 mLock;
|
| + int nFrame; /* Entries in aFrame[] array */
|
| + int nFrameAlloc; /* Allocated size of aFrame[] array */
|
| + RbuFrame *aFrame;
|
| + int pgsz;
|
| + u8 *aBuf;
|
| + i64 iWalCksum;
|
| +};
|
| +
|
| +/*
|
| +** An rbu VFS is implemented using an instance of this structure.
|
| +*/
|
| +struct rbu_vfs {
|
| + sqlite3_vfs base; /* rbu VFS shim methods */
|
| + sqlite3_vfs *pRealVfs; /* Underlying VFS */
|
| + sqlite3_mutex *mutex; /* Mutex to protect pMain */
|
| + rbu_file *pMain; /* Linked list of main db files */
|
| +};
|
| +
|
| +/*
|
| +** Each file opened by an rbu VFS is represented by an instance of
|
| +** the following structure.
|
| +*/
|
| +struct rbu_file {
|
| + sqlite3_file base; /* sqlite3_file methods */
|
| + sqlite3_file *pReal; /* Underlying file handle */
|
| + rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */
|
| + sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */
|
| +
|
| + int openFlags; /* Flags this file was opened with */
|
| + u32 iCookie; /* Cookie value for main db files */
|
| + u8 iWriteVer; /* "write-version" value for main db files */
|
| +
|
| + int nShm; /* Number of entries in apShm[] array */
|
| + char **apShm; /* Array of mmap'd *-shm regions */
|
| + char *zDel; /* Delete this when closing file */
|
| +
|
| + const char *zWal; /* Wal filename for this main db file */
|
| + rbu_file *pWalFd; /* Wal file descriptor for this main db */
|
| + rbu_file *pMainNext; /* Next MAIN_DB file */
|
| +};
|
| +
|
| +
|
| +/*************************************************************************
|
| +** The following three functions, found below:
|
| +**
|
| +** rbuDeltaGetInt()
|
| +** rbuDeltaChecksum()
|
| +** rbuDeltaApply()
|
| +**
|
| +** are lifted from the fossil source code (http://fossil-scm.org). They
|
| +** are used to implement the scalar SQL function rbu_fossil_delta().
|
| +*/
|
| +
|
| +/*
|
| +** Read bytes from *pz and convert them into a positive integer. When
|
| +** finished, leave *pz pointing to the first character past the end of
|
| +** the integer. The *pLen parameter holds the length of the string
|
| +** in *pz and is decremented once for each character in the integer.
|
| +*/
|
| +static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){
|
| + static const signed char zValue[] = {
|
| + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
| + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
| + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
| + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1,
|
| + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
|
| + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36,
|
| + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
|
| + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1,
|
| + };
|
| + unsigned int v = 0;
|
| + int c;
|
| + unsigned char *z = (unsigned char*)*pz;
|
| + unsigned char *zStart = z;
|
| + while( (c = zValue[0x7f&*(z++)])>=0 ){
|
| + v = (v<<6) + c;
|
| + }
|
| + z--;
|
| + *pLen -= z - zStart;
|
| + *pz = (char*)z;
|
| + return v;
|
| +}
|
| +
|
| +/*
|
| +** Compute a 32-bit checksum on the N-byte buffer. Return the result.
|
| +*/
|
| +static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){
|
| + const unsigned char *z = (const unsigned char *)zIn;
|
| + unsigned sum0 = 0;
|
| + unsigned sum1 = 0;
|
| + unsigned sum2 = 0;
|
| + unsigned sum3 = 0;
|
| + while(N >= 16){
|
| + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]);
|
| + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]);
|
| + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]);
|
| + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]);
|
| + z += 16;
|
| + N -= 16;
|
| + }
|
| + while(N >= 4){
|
| + sum0 += z[0];
|
| + sum1 += z[1];
|
| + sum2 += z[2];
|
| + sum3 += z[3];
|
| + z += 4;
|
| + N -= 4;
|
| + }
|
| + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24);
|
| + switch(N){
|
| + case 3: sum3 += (z[2] << 8);
|
| + case 2: sum3 += (z[1] << 16);
|
| + case 1: sum3 += (z[0] << 24);
|
| + default: ;
|
| + }
|
| + return sum3;
|
| +}
|
| +
|
| +/*
|
| +** Apply a delta.
|
| +**
|
| +** The output buffer should be big enough to hold the whole output
|
| +** file and a NUL terminator at the end. The delta_output_size()
|
| +** routine will determine this size for you.
|
| +**
|
| +** The delta string should be null-terminated. But the delta string
|
| +** may contain embedded NUL characters (if the input and output are
|
| +** binary files) so we also have to pass in the length of the delta in
|
| +** the lenDelta parameter.
|
| +**
|
| +** This function returns the size of the output file in bytes (excluding
|
| +** the final NUL terminator character). Except, if the delta string is
|
| +** malformed or intended for use with a source file other than zSrc,
|
| +** then this routine returns -1.
|
| +**
|
| +** Refer to the delta_create() documentation above for a description
|
| +** of the delta file format.
|
| +*/
|
| +static int rbuDeltaApply(
|
| + const char *zSrc, /* The source or pattern file */
|
| + int lenSrc, /* Length of the source file */
|
| + const char *zDelta, /* Delta to apply to the pattern */
|
| + int lenDelta, /* Length of the delta */
|
| + char *zOut /* Write the output into this preallocated buffer */
|
| +){
|
| + unsigned int limit;
|
| + unsigned int total = 0;
|
| +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST
|
| + char *zOrigOut = zOut;
|
| +#endif
|
| +
|
| + limit = rbuDeltaGetInt(&zDelta, &lenDelta);
|
| + if( *zDelta!='\n' ){
|
| + /* ERROR: size integer not terminated by "\n" */
|
| + return -1;
|
| + }
|
| + zDelta++; lenDelta--;
|
| + while( *zDelta && lenDelta>0 ){
|
| + unsigned int cnt, ofst;
|
| + cnt = rbuDeltaGetInt(&zDelta, &lenDelta);
|
| + switch( zDelta[0] ){
|
| + case '@': {
|
| + zDelta++; lenDelta--;
|
| + ofst = rbuDeltaGetInt(&zDelta, &lenDelta);
|
| + if( lenDelta>0 && zDelta[0]!=',' ){
|
| + /* ERROR: copy command not terminated by ',' */
|
| + return -1;
|
| + }
|
| + zDelta++; lenDelta--;
|
| + total += cnt;
|
| + if( total>limit ){
|
| + /* ERROR: copy exceeds output file size */
|
| + return -1;
|
| + }
|
| + if( (int)(ofst+cnt) > lenSrc ){
|
| + /* ERROR: copy extends past end of input */
|
| + return -1;
|
| + }
|
| + memcpy(zOut, &zSrc[ofst], cnt);
|
| + zOut += cnt;
|
| + break;
|
| + }
|
| + case ':': {
|
| + zDelta++; lenDelta--;
|
| + total += cnt;
|
| + if( total>limit ){
|
| + /* ERROR: insert command gives an output larger than predicted */
|
| + return -1;
|
| + }
|
| + if( (int)cnt>lenDelta ){
|
| + /* ERROR: insert count exceeds size of delta */
|
| + return -1;
|
| + }
|
| + memcpy(zOut, zDelta, cnt);
|
| + zOut += cnt;
|
| + zDelta += cnt;
|
| + lenDelta -= cnt;
|
| + break;
|
| + }
|
| + case ';': {
|
| + zDelta++; lenDelta--;
|
| + zOut[0] = 0;
|
| +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST
|
| + if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){
|
| + /* ERROR: bad checksum */
|
| + return -1;
|
| + }
|
| +#endif
|
| + if( total!=limit ){
|
| + /* ERROR: generated size does not match predicted size */
|
| + return -1;
|
| + }
|
| + return total;
|
| + }
|
| + default: {
|
| + /* ERROR: unknown delta operator */
|
| + return -1;
|
| + }
|
| + }
|
| + }
|
| + /* ERROR: unterminated delta */
|
| + return -1;
|
| +}
|
| +
|
| +static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){
|
| + int size;
|
| + size = rbuDeltaGetInt(&zDelta, &lenDelta);
|
| + if( *zDelta!='\n' ){
|
| + /* ERROR: size integer not terminated by "\n" */
|
| + return -1;
|
| + }
|
| + return size;
|
| +}
|
| +
|
| +/*
|
| +** End of code taken from fossil.
|
| +*************************************************************************/
|
| +
|
| +/*
|
| +** Implementation of SQL scalar function rbu_fossil_delta().
|
| +**
|
| +** This function applies a fossil delta patch to a blob. Exactly two
|
| +** arguments must be passed to this function. The first is the blob to
|
| +** patch and the second the patch to apply. If no error occurs, this
|
| +** function returns the patched blob.
|
| +*/
|
| +static void rbuFossilDeltaFunc(
|
| + sqlite3_context *context,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + const char *aDelta;
|
| + int nDelta;
|
| + const char *aOrig;
|
| + int nOrig;
|
| +
|
| + int nOut;
|
| + int nOut2;
|
| + char *aOut;
|
| +
|
| + assert( argc==2 );
|
| +
|
| + nOrig = sqlite3_value_bytes(argv[0]);
|
| + aOrig = (const char*)sqlite3_value_blob(argv[0]);
|
| + nDelta = sqlite3_value_bytes(argv[1]);
|
| + aDelta = (const char*)sqlite3_value_blob(argv[1]);
|
| +
|
| + /* Figure out the size of the output */
|
| + nOut = rbuDeltaOutputSize(aDelta, nDelta);
|
| + if( nOut<0 ){
|
| + sqlite3_result_error(context, "corrupt fossil delta", -1);
|
| + return;
|
| + }
|
| +
|
| + aOut = sqlite3_malloc(nOut+1);
|
| + if( aOut==0 ){
|
| + sqlite3_result_error_nomem(context);
|
| + }else{
|
| + nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut);
|
| + if( nOut2!=nOut ){
|
| + sqlite3_result_error(context, "corrupt fossil delta", -1);
|
| + }else{
|
| + sqlite3_result_blob(context, aOut, nOut, sqlite3_free);
|
| + }
|
| + }
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Prepare the SQL statement in buffer zSql against database handle db.
|
| +** If successful, set *ppStmt to point to the new statement and return
|
| +** SQLITE_OK.
|
| +**
|
| +** Otherwise, if an error does occur, set *ppStmt to NULL and return
|
| +** an SQLite error code. Additionally, set output variable *pzErrmsg to
|
| +** point to a buffer containing an error message. It is the responsibility
|
| +** of the caller to (eventually) free this buffer using sqlite3_free().
|
| +*/
|
| +static int prepareAndCollectError(
|
| + sqlite3 *db,
|
| + sqlite3_stmt **ppStmt,
|
| + char **pzErrmsg,
|
| + const char *zSql
|
| +){
|
| + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
|
| + if( rc!=SQLITE_OK ){
|
| + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
|
| + *ppStmt = 0;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Reset the SQL statement passed as the first argument. Return a copy
|
| +** of the value returned by sqlite3_reset().
|
| +**
|
| +** If an error has occurred, then set *pzErrmsg to point to a buffer
|
| +** containing an error message. It is the responsibility of the caller
|
| +** to eventually free this buffer using sqlite3_free().
|
| +*/
|
| +static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){
|
| + int rc = sqlite3_reset(pStmt);
|
| + if( rc!=SQLITE_OK ){
|
| + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt)));
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Unless it is NULL, argument zSql points to a buffer allocated using
|
| +** sqlite3_malloc containing an SQL statement. This function prepares the SQL
|
| +** statement against database db and frees the buffer. If statement
|
| +** compilation is successful, *ppStmt is set to point to the new statement
|
| +** handle and SQLITE_OK is returned.
|
| +**
|
| +** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code
|
| +** returned. In this case, *pzErrmsg may also be set to point to an error
|
| +** message. It is the responsibility of the caller to free this error message
|
| +** buffer using sqlite3_free().
|
| +**
|
| +** If argument zSql is NULL, this function assumes that an OOM has occurred.
|
| +** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL.
|
| +*/
|
| +static int prepareFreeAndCollectError(
|
| + sqlite3 *db,
|
| + sqlite3_stmt **ppStmt,
|
| + char **pzErrmsg,
|
| + char *zSql
|
| +){
|
| + int rc;
|
| + assert( *pzErrmsg==0 );
|
| + if( zSql==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + *ppStmt = 0;
|
| + }else{
|
| + rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql);
|
| + sqlite3_free(zSql);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated
|
| +** by an earlier call to rbuObjIterCacheTableInfo().
|
| +*/
|
| +static void rbuObjIterFreeCols(RbuObjIter *pIter){
|
| + int i;
|
| + for(i=0; i<pIter->nTblCol; i++){
|
| + sqlite3_free(pIter->azTblCol[i]);
|
| + sqlite3_free(pIter->azTblType[i]);
|
| + }
|
| + sqlite3_free(pIter->azTblCol);
|
| + pIter->azTblCol = 0;
|
| + pIter->azTblType = 0;
|
| + pIter->aiSrcOrder = 0;
|
| + pIter->abTblPk = 0;
|
| + pIter->abNotNull = 0;
|
| + pIter->nTblCol = 0;
|
| + pIter->eType = 0; /* Invalid value */
|
| +}
|
| +
|
| +/*
|
| +** Finalize all statements and free all allocations that are specific to
|
| +** the current object (table/index pair).
|
| +*/
|
| +static void rbuObjIterClearStatements(RbuObjIter *pIter){
|
| + RbuUpdateStmt *pUp;
|
| +
|
| + sqlite3_finalize(pIter->pSelect);
|
| + sqlite3_finalize(pIter->pInsert);
|
| + sqlite3_finalize(pIter->pDelete);
|
| + sqlite3_finalize(pIter->pTmpInsert);
|
| + pUp = pIter->pRbuUpdate;
|
| + while( pUp ){
|
| + RbuUpdateStmt *pTmp = pUp->pNext;
|
| + sqlite3_finalize(pUp->pUpdate);
|
| + sqlite3_free(pUp);
|
| + pUp = pTmp;
|
| + }
|
| +
|
| + pIter->pSelect = 0;
|
| + pIter->pInsert = 0;
|
| + pIter->pDelete = 0;
|
| + pIter->pRbuUpdate = 0;
|
| + pIter->pTmpInsert = 0;
|
| + pIter->nCol = 0;
|
| +}
|
| +
|
| +/*
|
| +** Clean up any resources allocated as part of the iterator object passed
|
| +** as the only argument.
|
| +*/
|
| +static void rbuObjIterFinalize(RbuObjIter *pIter){
|
| + rbuObjIterClearStatements(pIter);
|
| + sqlite3_finalize(pIter->pTblIter);
|
| + sqlite3_finalize(pIter->pIdxIter);
|
| + rbuObjIterFreeCols(pIter);
|
| + memset(pIter, 0, sizeof(RbuObjIter));
|
| +}
|
| +
|
| +/*
|
| +** Advance the iterator to the next position.
|
| +**
|
| +** If no error occurs, SQLITE_OK is returned and the iterator is left
|
| +** pointing to the next entry. Otherwise, an error code and message is
|
| +** left in the RBU handle passed as the first argument. A copy of the
|
| +** error code is returned.
|
| +*/
|
| +static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){
|
| + int rc = p->rc;
|
| + if( rc==SQLITE_OK ){
|
| +
|
| + /* Free any SQLite statements used while processing the previous object */
|
| + rbuObjIterClearStatements(pIter);
|
| + if( pIter->zIdx==0 ){
|
| + rc = sqlite3_exec(p->dbMain,
|
| + "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;"
|
| + "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;"
|
| + "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;"
|
| + "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;"
|
| + , 0, 0, &p->zErrmsg
|
| + );
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + if( pIter->bCleanup ){
|
| + rbuObjIterFreeCols(pIter);
|
| + pIter->bCleanup = 0;
|
| + rc = sqlite3_step(pIter->pTblIter);
|
| + if( rc!=SQLITE_ROW ){
|
| + rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg);
|
| + pIter->zTbl = 0;
|
| + }else{
|
| + pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0);
|
| + pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1);
|
| + rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM;
|
| + }
|
| + }else{
|
| + if( pIter->zIdx==0 ){
|
| + sqlite3_stmt *pIdx = pIter->pIdxIter;
|
| + rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3_step(pIter->pIdxIter);
|
| + if( rc!=SQLITE_ROW ){
|
| + rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg);
|
| + pIter->bCleanup = 1;
|
| + pIter->zIdx = 0;
|
| + }else{
|
| + pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0);
|
| + pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1);
|
| + pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2);
|
| + rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM;
|
| + }
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + rbuObjIterFinalize(pIter);
|
| + p->rc = rc;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** The implementation of the rbu_target_name() SQL function. This function
|
| +** accepts one argument - the name of a table in the RBU database. If the
|
| +** table name matches the pattern:
|
| +**
|
| +** data[0-9]_<name>
|
| +**
|
| +** where <name> is any sequence of 1 or more characters, <name> is returned.
|
| +** Otherwise, if the only argument does not match the above pattern, an SQL
|
| +** NULL is returned.
|
| +**
|
| +** "data_t1" -> "t1"
|
| +** "data0123_t2" -> "t2"
|
| +** "dataAB_t3" -> NULL
|
| +*/
|
| +static void rbuTargetNameFunc(
|
| + sqlite3_context *context,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + const char *zIn;
|
| + assert( argc==1 );
|
| +
|
| + zIn = (const char*)sqlite3_value_text(argv[0]);
|
| + if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){
|
| + int i;
|
| + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++);
|
| + if( zIn[i]=='_' && zIn[i+1] ){
|
| + sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC);
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Initialize the iterator structure passed as the second argument.
|
| +**
|
| +** If no error occurs, SQLITE_OK is returned and the iterator is left
|
| +** pointing to the first entry. Otherwise, an error code and message is
|
| +** left in the RBU handle passed as the first argument. A copy of the
|
| +** error code is returned.
|
| +*/
|
| +static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){
|
| + int rc;
|
| + memset(pIter, 0, sizeof(RbuObjIter));
|
| +
|
| + rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg,
|
| + "SELECT rbu_target_name(name) AS target, name FROM sqlite_master "
|
| + "WHERE type IN ('table', 'view') AND target IS NOT NULL "
|
| + "ORDER BY name"
|
| + );
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg,
|
| + "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' "
|
| + " FROM main.sqlite_master "
|
| + " WHERE type='index' AND tbl_name = ?"
|
| + );
|
| + }
|
| +
|
| + pIter->bCleanup = 1;
|
| + p->rc = rc;
|
| + return rbuObjIterNext(p, pIter);
|
| +}
|
| +
|
| +/*
|
| +** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs,
|
| +** an error code is stored in the RBU handle passed as the first argument.
|
| +**
|
| +** If an error has already occurred (p->rc is already set to something other
|
| +** than SQLITE_OK), then this function returns NULL without modifying the
|
| +** stored error code. In this case it still calls sqlite3_free() on any
|
| +** printf() parameters associated with %z conversions.
|
| +*/
|
| +static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){
|
| + char *zSql = 0;
|
| + va_list ap;
|
| + va_start(ap, zFmt);
|
| + zSql = sqlite3_vmprintf(zFmt, ap);
|
| + if( p->rc==SQLITE_OK ){
|
| + if( zSql==0 ) p->rc = SQLITE_NOMEM;
|
| + }else{
|
| + sqlite3_free(zSql);
|
| + zSql = 0;
|
| + }
|
| + va_end(ap);
|
| + return zSql;
|
| +}
|
| +
|
| +/*
|
| +** Argument zFmt is a sqlite3_mprintf() style format string. The trailing
|
| +** arguments are the usual subsitution values. This function performs
|
| +** the printf() style substitutions and executes the result as an SQL
|
| +** statement on the RBU handles database.
|
| +**
|
| +** If an error occurs, an error code and error message is stored in the
|
| +** RBU handle. If an error has already occurred when this function is
|
| +** called, it is a no-op.
|
| +*/
|
| +static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){
|
| + va_list ap;
|
| + char *zSql;
|
| + va_start(ap, zFmt);
|
| + zSql = sqlite3_vmprintf(zFmt, ap);
|
| + if( p->rc==SQLITE_OK ){
|
| + if( zSql==0 ){
|
| + p->rc = SQLITE_NOMEM;
|
| + }else{
|
| + p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg);
|
| + }
|
| + }
|
| + sqlite3_free(zSql);
|
| + va_end(ap);
|
| + return p->rc;
|
| +}
|
| +
|
| +/*
|
| +** Attempt to allocate and return a pointer to a zeroed block of nByte
|
| +** bytes.
|
| +**
|
| +** If an error (i.e. an OOM condition) occurs, return NULL and leave an
|
| +** error code in the rbu handle passed as the first argument. Or, if an
|
| +** error has already occurred when this function is called, return NULL
|
| +** immediately without attempting the allocation or modifying the stored
|
| +** error code.
|
| +*/
|
| +static void *rbuMalloc(sqlite3rbu *p, int nByte){
|
| + void *pRet = 0;
|
| + if( p->rc==SQLITE_OK ){
|
| + assert( nByte>0 );
|
| + pRet = sqlite3_malloc(nByte);
|
| + if( pRet==0 ){
|
| + p->rc = SQLITE_NOMEM;
|
| + }else{
|
| + memset(pRet, 0, nByte);
|
| + }
|
| + }
|
| + return pRet;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that
|
| +** there is room for at least nCol elements. If an OOM occurs, store an
|
| +** error code in the RBU handle passed as the first argument.
|
| +*/
|
| +static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){
|
| + int nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol;
|
| + char **azNew;
|
| +
|
| + azNew = (char**)rbuMalloc(p, nByte);
|
| + if( azNew ){
|
| + pIter->azTblCol = azNew;
|
| + pIter->azTblType = &azNew[nCol];
|
| + pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol];
|
| + pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol];
|
| + pIter->abNotNull = (u8*)&pIter->abTblPk[nCol];
|
| + pIter->abIndexed = (u8*)&pIter->abNotNull[nCol];
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** The first argument must be a nul-terminated string. This function
|
| +** returns a copy of the string in memory obtained from sqlite3_malloc().
|
| +** It is the responsibility of the caller to eventually free this memory
|
| +** using sqlite3_free().
|
| +**
|
| +** If an OOM condition is encountered when attempting to allocate memory,
|
| +** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise,
|
| +** if the allocation succeeds, (*pRc) is left unchanged.
|
| +*/
|
| +static char *rbuStrndup(const char *zStr, int *pRc){
|
| + char *zRet = 0;
|
| +
|
| + assert( *pRc==SQLITE_OK );
|
| + if( zStr ){
|
| + int nCopy = strlen(zStr) + 1;
|
| + zRet = (char*)sqlite3_malloc(nCopy);
|
| + if( zRet ){
|
| + memcpy(zRet, zStr, nCopy);
|
| + }else{
|
| + *pRc = SQLITE_NOMEM;
|
| + }
|
| + }
|
| +
|
| + return zRet;
|
| +}
|
| +
|
| +/*
|
| +** Finalize the statement passed as the second argument.
|
| +**
|
| +** If the sqlite3_finalize() call indicates that an error occurs, and the
|
| +** rbu handle error code is not already set, set the error code and error
|
| +** message accordingly.
|
| +*/
|
| +static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){
|
| + sqlite3 *db = sqlite3_db_handle(pStmt);
|
| + int rc = sqlite3_finalize(pStmt);
|
| + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){
|
| + p->rc = rc;
|
| + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
|
| + }
|
| +}
|
| +
|
| +/* Determine the type of a table.
|
| +**
|
| +** peType is of type (int*), a pointer to an output parameter of type
|
| +** (int). This call sets the output parameter as follows, depending
|
| +** on the type of the table specified by parameters dbName and zTbl.
|
| +**
|
| +** RBU_PK_NOTABLE: No such table.
|
| +** RBU_PK_NONE: Table has an implicit rowid.
|
| +** RBU_PK_IPK: Table has an explicit IPK column.
|
| +** RBU_PK_EXTERNAL: Table has an external PK index.
|
| +** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID.
|
| +** RBU_PK_VTAB: Table is a virtual table.
|
| +**
|
| +** Argument *piPk is also of type (int*), and also points to an output
|
| +** parameter. Unless the table has an external primary key index
|
| +** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or,
|
| +** if the table does have an external primary key index, then *piPk
|
| +** is set to the root page number of the primary key index before
|
| +** returning.
|
| +**
|
| +** ALGORITHM:
|
| +**
|
| +** if( no entry exists in sqlite_master ){
|
| +** return RBU_PK_NOTABLE
|
| +** }else if( sql for the entry starts with "CREATE VIRTUAL" ){
|
| +** return RBU_PK_VTAB
|
| +** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){
|
| +** if( the index that is the pk exists in sqlite_master ){
|
| +** *piPK = rootpage of that index.
|
| +** return RBU_PK_EXTERNAL
|
| +** }else{
|
| +** return RBU_PK_WITHOUT_ROWID
|
| +** }
|
| +** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){
|
| +** return RBU_PK_IPK
|
| +** }else{
|
| +** return RBU_PK_NONE
|
| +** }
|
| +*/
|
| +static void rbuTableType(
|
| + sqlite3rbu *p,
|
| + const char *zTab,
|
| + int *peType,
|
| + int *piTnum,
|
| + int *piPk
|
| +){
|
| + /*
|
| + ** 0) SELECT count(*) FROM sqlite_master where name=%Q AND IsVirtual(%Q)
|
| + ** 1) PRAGMA index_list = ?
|
| + ** 2) SELECT count(*) FROM sqlite_master where name=%Q
|
| + ** 3) PRAGMA table_info = ?
|
| + */
|
| + sqlite3_stmt *aStmt[4] = {0, 0, 0, 0};
|
| +
|
| + *peType = RBU_PK_NOTABLE;
|
| + *piPk = 0;
|
| +
|
| + assert( p->rc==SQLITE_OK );
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg,
|
| + sqlite3_mprintf(
|
| + "SELECT (sql LIKE 'create virtual%%'), rootpage"
|
| + " FROM sqlite_master"
|
| + " WHERE name=%Q", zTab
|
| + ));
|
| + if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){
|
| + /* Either an error, or no such table. */
|
| + goto rbuTableType_end;
|
| + }
|
| + if( sqlite3_column_int(aStmt[0], 0) ){
|
| + *peType = RBU_PK_VTAB; /* virtual table */
|
| + goto rbuTableType_end;
|
| + }
|
| + *piTnum = sqlite3_column_int(aStmt[0], 1);
|
| +
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg,
|
| + sqlite3_mprintf("PRAGMA index_list=%Q",zTab)
|
| + );
|
| + if( p->rc ) goto rbuTableType_end;
|
| + while( sqlite3_step(aStmt[1])==SQLITE_ROW ){
|
| + const u8 *zOrig = sqlite3_column_text(aStmt[1], 3);
|
| + const u8 *zIdx = sqlite3_column_text(aStmt[1], 1);
|
| + if( zOrig && zIdx && zOrig[0]=='p' ){
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg,
|
| + sqlite3_mprintf(
|
| + "SELECT rootpage FROM sqlite_master WHERE name = %Q", zIdx
|
| + ));
|
| + if( p->rc==SQLITE_OK ){
|
| + if( sqlite3_step(aStmt[2])==SQLITE_ROW ){
|
| + *piPk = sqlite3_column_int(aStmt[2], 0);
|
| + *peType = RBU_PK_EXTERNAL;
|
| + }else{
|
| + *peType = RBU_PK_WITHOUT_ROWID;
|
| + }
|
| + }
|
| + goto rbuTableType_end;
|
| + }
|
| + }
|
| +
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg,
|
| + sqlite3_mprintf("PRAGMA table_info=%Q",zTab)
|
| + );
|
| + if( p->rc==SQLITE_OK ){
|
| + while( sqlite3_step(aStmt[3])==SQLITE_ROW ){
|
| + if( sqlite3_column_int(aStmt[3],5)>0 ){
|
| + *peType = RBU_PK_IPK; /* explicit IPK column */
|
| + goto rbuTableType_end;
|
| + }
|
| + }
|
| + *peType = RBU_PK_NONE;
|
| + }
|
| +
|
| +rbuTableType_end: {
|
| + unsigned int i;
|
| + for(i=0; i<sizeof(aStmt)/sizeof(aStmt[0]); i++){
|
| + rbuFinalize(p, aStmt[i]);
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** This is a helper function for rbuObjIterCacheTableInfo(). It populates
|
| +** the pIter->abIndexed[] array.
|
| +*/
|
| +static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){
|
| + sqlite3_stmt *pList = 0;
|
| + int bIndex = 0;
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol);
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg,
|
| + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl)
|
| + );
|
| + }
|
| +
|
| + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){
|
| + const char *zIdx = (const char*)sqlite3_column_text(pList, 1);
|
| + sqlite3_stmt *pXInfo = 0;
|
| + if( zIdx==0 ) break;
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
|
| + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
|
| + );
|
| + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
|
| + int iCid = sqlite3_column_int(pXInfo, 1);
|
| + if( iCid>=0 ) pIter->abIndexed[iCid] = 1;
|
| + }
|
| + rbuFinalize(p, pXInfo);
|
| + bIndex = 1;
|
| + }
|
| +
|
| + rbuFinalize(p, pList);
|
| + if( bIndex==0 ) pIter->abIndexed = 0;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** If they are not already populated, populate the pIter->azTblCol[],
|
| +** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to
|
| +** the table (not index) that the iterator currently points to.
|
| +**
|
| +** Return SQLITE_OK if successful, or an SQLite error code otherwise. If
|
| +** an error does occur, an error code and error message are also left in
|
| +** the RBU handle.
|
| +*/
|
| +static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){
|
| + if( pIter->azTblCol==0 ){
|
| + sqlite3_stmt *pStmt = 0;
|
| + int nCol = 0;
|
| + int i; /* for() loop iterator variable */
|
| + int bRbuRowid = 0; /* If input table has column "rbu_rowid" */
|
| + int iOrder = 0;
|
| + int iTnum = 0;
|
| +
|
| + /* Figure out the type of table this step will deal with. */
|
| + assert( pIter->eType==0 );
|
| + rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum);
|
| + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){
|
| + p->rc = SQLITE_ERROR;
|
| + p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl);
|
| + }
|
| + if( p->rc ) return p->rc;
|
| + if( pIter->zIdx==0 ) pIter->iTnum = iTnum;
|
| +
|
| + assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK
|
| + || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID
|
| + || pIter->eType==RBU_PK_VTAB
|
| + );
|
| +
|
| + /* Populate the azTblCol[] and nTblCol variables based on the columns
|
| + ** of the input table. Ignore any input table columns that begin with
|
| + ** "rbu_". */
|
| + p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
|
| + sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl)
|
| + );
|
| + if( p->rc==SQLITE_OK ){
|
| + nCol = sqlite3_column_count(pStmt);
|
| + rbuAllocateIterArrays(p, pIter, nCol);
|
| + }
|
| + for(i=0; p->rc==SQLITE_OK && i<nCol; i++){
|
| + const char *zName = (const char*)sqlite3_column_name(pStmt, i);
|
| + if( sqlite3_strnicmp("rbu_", zName, 4) ){
|
| + char *zCopy = rbuStrndup(zName, &p->rc);
|
| + pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol;
|
| + pIter->azTblCol[pIter->nTblCol++] = zCopy;
|
| + }
|
| + else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){
|
| + bRbuRowid = 1;
|
| + }
|
| + }
|
| + sqlite3_finalize(pStmt);
|
| + pStmt = 0;
|
| +
|
| + if( p->rc==SQLITE_OK
|
| + && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE)
|
| + ){
|
| + p->rc = SQLITE_ERROR;
|
| + p->zErrmsg = sqlite3_mprintf(
|
| + "table %q %s rbu_rowid column", pIter->zDataTbl,
|
| + (bRbuRowid ? "may not have" : "requires")
|
| + );
|
| + }
|
| +
|
| + /* Check that all non-HIDDEN columns in the destination table are also
|
| + ** present in the input table. Populate the abTblPk[], azTblType[] and
|
| + ** aiTblOrder[] arrays at the same time. */
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg,
|
| + sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl)
|
| + );
|
| + }
|
| + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
|
| + const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
|
| + if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */
|
| + for(i=iOrder; i<pIter->nTblCol; i++){
|
| + if( 0==strcmp(zName, pIter->azTblCol[i]) ) break;
|
| + }
|
| + if( i==pIter->nTblCol ){
|
| + p->rc = SQLITE_ERROR;
|
| + p->zErrmsg = sqlite3_mprintf("column missing from %q: %s",
|
| + pIter->zDataTbl, zName
|
| + );
|
| + }else{
|
| + int iPk = sqlite3_column_int(pStmt, 5);
|
| + int bNotNull = sqlite3_column_int(pStmt, 3);
|
| + const char *zType = (const char*)sqlite3_column_text(pStmt, 2);
|
| +
|
| + if( i!=iOrder ){
|
| + SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]);
|
| + SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]);
|
| + }
|
| +
|
| + pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc);
|
| + pIter->abTblPk[iOrder] = (iPk!=0);
|
| + pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0);
|
| + iOrder++;
|
| + }
|
| + }
|
| +
|
| + rbuFinalize(p, pStmt);
|
| + rbuObjIterCacheIndexedCols(p, pIter);
|
| + assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 );
|
| + }
|
| +
|
| + return p->rc;
|
| +}
|
| +
|
| +/*
|
| +** This function constructs and returns a pointer to a nul-terminated
|
| +** string containing some SQL clause or list based on one or more of the
|
| +** column names currently stored in the pIter->azTblCol[] array.
|
| +*/
|
| +static char *rbuObjIterGetCollist(
|
| + sqlite3rbu *p, /* RBU object */
|
| + RbuObjIter *pIter /* Object iterator for column names */
|
| +){
|
| + char *zList = 0;
|
| + const char *zSep = "";
|
| + int i;
|
| + for(i=0; i<pIter->nTblCol; i++){
|
| + const char *z = pIter->azTblCol[i];
|
| + zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z);
|
| + zSep = ", ";
|
| + }
|
| + return zList;
|
| +}
|
| +
|
| +/*
|
| +** This function is used to create a SELECT list (the list of SQL
|
| +** expressions that follows a SELECT keyword) for a SELECT statement
|
| +** used to read from an data_xxx or rbu_tmp_xxx table while updating the
|
| +** index object currently indicated by the iterator object passed as the
|
| +** second argument. A "PRAGMA index_xinfo = <idxname>" statement is used
|
| +** to obtain the required information.
|
| +**
|
| +** If the index is of the following form:
|
| +**
|
| +** CREATE INDEX i1 ON t1(c, b COLLATE nocase);
|
| +**
|
| +** and "t1" is a table with an explicit INTEGER PRIMARY KEY column
|
| +** "ipk", the returned string is:
|
| +**
|
| +** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'"
|
| +**
|
| +** As well as the returned string, three other malloc'd strings are
|
| +** returned via output parameters. As follows:
|
| +**
|
| +** pzImposterCols: ...
|
| +** pzImposterPk: ...
|
| +** pzWhere: ...
|
| +*/
|
| +static char *rbuObjIterGetIndexCols(
|
| + sqlite3rbu *p, /* RBU object */
|
| + RbuObjIter *pIter, /* Object iterator for column names */
|
| + char **pzImposterCols, /* OUT: Columns for imposter table */
|
| + char **pzImposterPk, /* OUT: Imposter PK clause */
|
| + char **pzWhere, /* OUT: WHERE clause */
|
| + int *pnBind /* OUT: Trbul number of columns */
|
| +){
|
| + int rc = p->rc; /* Error code */
|
| + int rc2; /* sqlite3_finalize() return code */
|
| + char *zRet = 0; /* String to return */
|
| + char *zImpCols = 0; /* String to return via *pzImposterCols */
|
| + char *zImpPK = 0; /* String to return via *pzImposterPK */
|
| + char *zWhere = 0; /* String to return via *pzWhere */
|
| + int nBind = 0; /* Value to return via *pnBind */
|
| + const char *zCom = ""; /* Set to ", " later on */
|
| + const char *zAnd = ""; /* Set to " AND " later on */
|
| + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + assert( p->zErrmsg==0 );
|
| + rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
|
| + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx)
|
| + );
|
| + }
|
| +
|
| + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
|
| + int iCid = sqlite3_column_int(pXInfo, 1);
|
| + int bDesc = sqlite3_column_int(pXInfo, 3);
|
| + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
|
| + const char *zCol;
|
| + const char *zType;
|
| +
|
| + if( iCid<0 ){
|
| + /* An integer primary key. If the table has an explicit IPK, use
|
| + ** its name. Otherwise, use "rbu_rowid". */
|
| + if( pIter->eType==RBU_PK_IPK ){
|
| + int i;
|
| + for(i=0; pIter->abTblPk[i]==0; i++);
|
| + assert( i<pIter->nTblCol );
|
| + zCol = pIter->azTblCol[i];
|
| + }else{
|
| + zCol = "rbu_rowid";
|
| + }
|
| + zType = "INTEGER";
|
| + }else{
|
| + zCol = pIter->azTblCol[iCid];
|
| + zType = pIter->azTblType[iCid];
|
| + }
|
| +
|
| + zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate);
|
| + if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){
|
| + const char *zOrder = (bDesc ? " DESC" : "");
|
| + zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s",
|
| + zImpPK, zCom, nBind, zCol, zOrder
|
| + );
|
| + }
|
| + zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q",
|
| + zImpCols, zCom, nBind, zCol, zType, zCollate
|
| + );
|
| + zWhere = sqlite3_mprintf(
|
| + "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol
|
| + );
|
| + if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM;
|
| + zCom = ", ";
|
| + zAnd = " AND ";
|
| + nBind++;
|
| + }
|
| +
|
| + rc2 = sqlite3_finalize(pXInfo);
|
| + if( rc==SQLITE_OK ) rc = rc2;
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + sqlite3_free(zRet);
|
| + sqlite3_free(zImpCols);
|
| + sqlite3_free(zImpPK);
|
| + sqlite3_free(zWhere);
|
| + zRet = 0;
|
| + zImpCols = 0;
|
| + zImpPK = 0;
|
| + zWhere = 0;
|
| + p->rc = rc;
|
| + }
|
| +
|
| + *pzImposterCols = zImpCols;
|
| + *pzImposterPk = zImpPK;
|
| + *pzWhere = zWhere;
|
| + *pnBind = nBind;
|
| + return zRet;
|
| +}
|
| +
|
| +/*
|
| +** Assuming the current table columns are "a", "b" and "c", and the zObj
|
| +** paramter is passed "old", return a string of the form:
|
| +**
|
| +** "old.a, old.b, old.b"
|
| +**
|
| +** With the column names escaped.
|
| +**
|
| +** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append
|
| +** the text ", old._rowid_" to the returned value.
|
| +*/
|
| +static char *rbuObjIterGetOldlist(
|
| + sqlite3rbu *p,
|
| + RbuObjIter *pIter,
|
| + const char *zObj
|
| +){
|
| + char *zList = 0;
|
| + if( p->rc==SQLITE_OK && pIter->abIndexed ){
|
| + const char *zS = "";
|
| + int i;
|
| + for(i=0; i<pIter->nTblCol; i++){
|
| + if( pIter->abIndexed[i] ){
|
| + const char *zCol = pIter->azTblCol[i];
|
| + zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol);
|
| + }else{
|
| + zList = sqlite3_mprintf("%z%sNULL", zList, zS);
|
| + }
|
| + zS = ", ";
|
| + if( zList==0 ){
|
| + p->rc = SQLITE_NOMEM;
|
| + break;
|
| + }
|
| + }
|
| +
|
| + /* For a table with implicit rowids, append "old._rowid_" to the list. */
|
| + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
|
| + zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj);
|
| + }
|
| + }
|
| + return zList;
|
| +}
|
| +
|
| +/*
|
| +** Return an expression that can be used in a WHERE clause to match the
|
| +** primary key of the current table. For example, if the table is:
|
| +**
|
| +** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c));
|
| +**
|
| +** Return the string:
|
| +**
|
| +** "b = ?1 AND c = ?2"
|
| +*/
|
| +static char *rbuObjIterGetWhere(
|
| + sqlite3rbu *p,
|
| + RbuObjIter *pIter
|
| +){
|
| + char *zList = 0;
|
| + if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){
|
| + zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1);
|
| + }else if( pIter->eType==RBU_PK_EXTERNAL ){
|
| + const char *zSep = "";
|
| + int i;
|
| + for(i=0; i<pIter->nTblCol; i++){
|
| + if( pIter->abTblPk[i] ){
|
| + zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1);
|
| + zSep = " AND ";
|
| + }
|
| + }
|
| + zList = rbuMPrintf(p,
|
| + "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList
|
| + );
|
| +
|
| + }else{
|
| + const char *zSep = "";
|
| + int i;
|
| + for(i=0; i<pIter->nTblCol; i++){
|
| + if( pIter->abTblPk[i] ){
|
| + const char *zCol = pIter->azTblCol[i];
|
| + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1);
|
| + zSep = " AND ";
|
| + }
|
| + }
|
| + }
|
| + return zList;
|
| +}
|
| +
|
| +/*
|
| +** The SELECT statement iterating through the keys for the current object
|
| +** (p->objiter.pSelect) currently points to a valid row. However, there
|
| +** is something wrong with the rbu_control value in the rbu_control value
|
| +** stored in the (p->nCol+1)'th column. Set the error code and error message
|
| +** of the RBU handle to something reflecting this.
|
| +*/
|
| +static void rbuBadControlError(sqlite3rbu *p){
|
| + p->rc = SQLITE_ERROR;
|
| + p->zErrmsg = sqlite3_mprintf("invalid rbu_control value");
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Return a nul-terminated string containing the comma separated list of
|
| +** assignments that should be included following the "SET" keyword of
|
| +** an UPDATE statement used to update the table object that the iterator
|
| +** passed as the second argument currently points to if the rbu_control
|
| +** column of the data_xxx table entry is set to zMask.
|
| +**
|
| +** The memory for the returned string is obtained from sqlite3_malloc().
|
| +** It is the responsibility of the caller to eventually free it using
|
| +** sqlite3_free().
|
| +**
|
| +** If an OOM error is encountered when allocating space for the new
|
| +** string, an error code is left in the rbu handle passed as the first
|
| +** argument and NULL is returned. Or, if an error has already occurred
|
| +** when this function is called, NULL is returned immediately, without
|
| +** attempting the allocation or modifying the stored error code.
|
| +*/
|
| +static char *rbuObjIterGetSetlist(
|
| + sqlite3rbu *p,
|
| + RbuObjIter *pIter,
|
| + const char *zMask
|
| +){
|
| + char *zList = 0;
|
| + if( p->rc==SQLITE_OK ){
|
| + int i;
|
| +
|
| + if( (int)strlen(zMask)!=pIter->nTblCol ){
|
| + rbuBadControlError(p);
|
| + }else{
|
| + const char *zSep = "";
|
| + for(i=0; i<pIter->nTblCol; i++){
|
| + char c = zMask[pIter->aiSrcOrder[i]];
|
| + if( c=='x' ){
|
| + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d",
|
| + zList, zSep, pIter->azTblCol[i], i+1
|
| + );
|
| + zSep = ", ";
|
| + }
|
| + else if( c=='d' ){
|
| + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)",
|
| + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1
|
| + );
|
| + zSep = ", ";
|
| + }
|
| + else if( c=='f' ){
|
| + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)",
|
| + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1
|
| + );
|
| + zSep = ", ";
|
| + }
|
| + }
|
| + }
|
| + }
|
| + return zList;
|
| +}
|
| +
|
| +/*
|
| +** Return a nul-terminated string consisting of nByte comma separated
|
| +** "?" expressions. For example, if nByte is 3, return a pointer to
|
| +** a buffer containing the string "?,?,?".
|
| +**
|
| +** The memory for the returned string is obtained from sqlite3_malloc().
|
| +** It is the responsibility of the caller to eventually free it using
|
| +** sqlite3_free().
|
| +**
|
| +** If an OOM error is encountered when allocating space for the new
|
| +** string, an error code is left in the rbu handle passed as the first
|
| +** argument and NULL is returned. Or, if an error has already occurred
|
| +** when this function is called, NULL is returned immediately, without
|
| +** attempting the allocation or modifying the stored error code.
|
| +*/
|
| +static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){
|
| + char *zRet = 0;
|
| + int nByte = nBind*2 + 1;
|
| +
|
| + zRet = (char*)rbuMalloc(p, nByte);
|
| + if( zRet ){
|
| + int i;
|
| + for(i=0; i<nBind; i++){
|
| + zRet[i*2] = '?';
|
| + zRet[i*2+1] = (i+1==nBind) ? '\0' : ',';
|
| + }
|
| + }
|
| + return zRet;
|
| +}
|
| +
|
| +/*
|
| +** The iterator currently points to a table (not index) of type
|
| +** RBU_PK_WITHOUT_ROWID. This function creates the PRIMARY KEY
|
| +** declaration for the corresponding imposter table. For example,
|
| +** if the iterator points to a table created as:
|
| +**
|
| +** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, a DESC)) WITHOUT ROWID
|
| +**
|
| +** this function returns:
|
| +**
|
| +** PRIMARY KEY("b", "a" DESC)
|
| +*/
|
| +static char *rbuWithoutRowidPK(sqlite3rbu *p, RbuObjIter *pIter){
|
| + char *z = 0;
|
| + assert( pIter->zIdx==0 );
|
| + if( p->rc==SQLITE_OK ){
|
| + const char *zSep = "PRIMARY KEY(";
|
| + sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */
|
| + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = <pk-index> */
|
| +
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg,
|
| + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl)
|
| + );
|
| + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){
|
| + const char *zOrig = (const char*)sqlite3_column_text(pXList,3);
|
| + if( zOrig && strcmp(zOrig, "pk")==0 ){
|
| + const char *zIdx = (const char*)sqlite3_column_text(pXList,1);
|
| + if( zIdx ){
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
|
| + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
|
| + );
|
| + }
|
| + break;
|
| + }
|
| + }
|
| + rbuFinalize(p, pXList);
|
| +
|
| + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
|
| + if( sqlite3_column_int(pXInfo, 5) ){
|
| + /* int iCid = sqlite3_column_int(pXInfo, 0); */
|
| + const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2);
|
| + const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : "";
|
| + z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc);
|
| + zSep = ", ";
|
| + }
|
| + }
|
| + z = rbuMPrintf(p, "%z)", z);
|
| + rbuFinalize(p, pXInfo);
|
| + }
|
| + return z;
|
| +}
|
| +
|
| +/*
|
| +** This function creates the second imposter table used when writing to
|
| +** a table b-tree where the table has an external primary key. If the
|
| +** iterator passed as the second argument does not currently point to
|
| +** a table (not index) with an external primary key, this function is a
|
| +** no-op.
|
| +**
|
| +** Assuming the iterator does point to a table with an external PK, this
|
| +** function creates a WITHOUT ROWID imposter table named "rbu_imposter2"
|
| +** used to access that PK index. For example, if the target table is
|
| +** declared as follows:
|
| +**
|
| +** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c));
|
| +**
|
| +** then the imposter table schema is:
|
| +**
|
| +** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID;
|
| +**
|
| +*/
|
| +static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){
|
| + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){
|
| + int tnum = pIter->iPkTnum; /* Root page of PK index */
|
| + sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */
|
| + const char *zIdx = 0; /* Name of PK index */
|
| + sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */
|
| + const char *zComma = "";
|
| + char *zCols = 0; /* Used to build up list of table cols */
|
| + char *zPk = 0; /* Used to build up table PK declaration */
|
| +
|
| + /* Figure out the name of the primary key index for the current table.
|
| + ** This is needed for the argument to "PRAGMA index_xinfo". Set
|
| + ** zIdx to point to a nul-terminated string containing this name. */
|
| + p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg,
|
| + "SELECT name FROM sqlite_master WHERE rootpage = ?"
|
| + );
|
| + if( p->rc==SQLITE_OK ){
|
| + sqlite3_bind_int(pQuery, 1, tnum);
|
| + if( SQLITE_ROW==sqlite3_step(pQuery) ){
|
| + zIdx = (const char*)sqlite3_column_text(pQuery, 0);
|
| + }
|
| + }
|
| + if( zIdx ){
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
|
| + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
|
| + );
|
| + }
|
| + rbuFinalize(p, pQuery);
|
| +
|
| + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
|
| + int bKey = sqlite3_column_int(pXInfo, 5);
|
| + if( bKey ){
|
| + int iCid = sqlite3_column_int(pXInfo, 1);
|
| + int bDesc = sqlite3_column_int(pXInfo, 3);
|
| + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
|
| + zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %s", zCols, zComma,
|
| + iCid, pIter->azTblType[iCid], zCollate
|
| + );
|
| + zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":"");
|
| + zComma = ", ";
|
| + }
|
| + }
|
| + zCols = rbuMPrintf(p, "%z, id INTEGER", zCols);
|
| + rbuFinalize(p, pXInfo);
|
| +
|
| + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum);
|
| + rbuMPrintfExec(p, p->dbMain,
|
| + "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID",
|
| + zCols, zPk
|
| + );
|
| + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** If an error has already occurred when this function is called, it
|
| +** immediately returns zero (without doing any work). Or, if an error
|
| +** occurs during the execution of this function, it sets the error code
|
| +** in the sqlite3rbu object indicated by the first argument and returns
|
| +** zero.
|
| +**
|
| +** The iterator passed as the second argument is guaranteed to point to
|
| +** a table (not an index) when this function is called. This function
|
| +** attempts to create any imposter table required to write to the main
|
| +** table b-tree of the table before returning. Non-zero is returned if
|
| +** an imposter table are created, or zero otherwise.
|
| +**
|
| +** An imposter table is required in all cases except RBU_PK_VTAB. Only
|
| +** virtual tables are written to directly. The imposter table has the
|
| +** same schema as the actual target table (less any UNIQUE constraints).
|
| +** More precisely, the "same schema" means the same columns, types,
|
| +** collation sequences. For tables that do not have an external PRIMARY
|
| +** KEY, it also means the same PRIMARY KEY declaration.
|
| +*/
|
| +static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){
|
| + if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){
|
| + int tnum = pIter->iTnum;
|
| + const char *zComma = "";
|
| + char *zSql = 0;
|
| + int iCol;
|
| + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1);
|
| +
|
| + for(iCol=0; p->rc==SQLITE_OK && iCol<pIter->nTblCol; iCol++){
|
| + const char *zPk = "";
|
| + const char *zCol = pIter->azTblCol[iCol];
|
| + const char *zColl = 0;
|
| +
|
| + p->rc = sqlite3_table_column_metadata(
|
| + p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0
|
| + );
|
| +
|
| + if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){
|
| + /* If the target table column is an "INTEGER PRIMARY KEY", add
|
| + ** "PRIMARY KEY" to the imposter table column declaration. */
|
| + zPk = "PRIMARY KEY ";
|
| + }
|
| + zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s%s",
|
| + zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl,
|
| + (pIter->abNotNull[iCol] ? " NOT NULL" : "")
|
| + );
|
| + zComma = ", ";
|
| + }
|
| +
|
| + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){
|
| + char *zPk = rbuWithoutRowidPK(p, pIter);
|
| + if( zPk ){
|
| + zSql = rbuMPrintf(p, "%z, %z", zSql, zPk);
|
| + }
|
| + }
|
| +
|
| + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum);
|
| + rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s",
|
| + pIter->zTbl, zSql,
|
| + (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "")
|
| + );
|
| + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Prepare a statement used to insert rows into the "rbu_tmp_xxx" table.
|
| +** Specifically a statement of the form:
|
| +**
|
| +** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...);
|
| +**
|
| +** The number of bound variables is equal to the number of columns in
|
| +** the target table, plus one (for the rbu_control column), plus one more
|
| +** (for the rbu_rowid column) if the target table is an implicit IPK or
|
| +** virtual table.
|
| +*/
|
| +static void rbuObjIterPrepareTmpInsert(
|
| + sqlite3rbu *p,
|
| + RbuObjIter *pIter,
|
| + const char *zCollist,
|
| + const char *zRbuRowid
|
| +){
|
| + int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE);
|
| + char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid);
|
| + if( zBind ){
|
| + assert( pIter->pTmpInsert==0 );
|
| + p->rc = prepareFreeAndCollectError(
|
| + p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf(
|
| + "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)",
|
| + p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind
|
| + ));
|
| + }
|
| +}
|
| +
|
| +static void rbuTmpInsertFunc(
|
| + sqlite3_context *pCtx,
|
| + int nVal,
|
| + sqlite3_value **apVal
|
| +){
|
| + sqlite3rbu *p = sqlite3_user_data(pCtx);
|
| + int rc = SQLITE_OK;
|
| + int i;
|
| +
|
| + for(i=0; rc==SQLITE_OK && i<nVal; i++){
|
| + rc = sqlite3_bind_value(p->objiter.pTmpInsert, i+1, apVal[i]);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_step(p->objiter.pTmpInsert);
|
| + rc = sqlite3_reset(p->objiter.pTmpInsert);
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + sqlite3_result_error_code(pCtx, rc);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Ensure that the SQLite statement handles required to update the
|
| +** target database object currently indicated by the iterator passed
|
| +** as the second argument are available.
|
| +*/
|
| +static int rbuObjIterPrepareAll(
|
| + sqlite3rbu *p,
|
| + RbuObjIter *pIter,
|
| + int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */
|
| +){
|
| + assert( pIter->bCleanup==0 );
|
| + if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){
|
| + const int tnum = pIter->iTnum;
|
| + char *zCollist = 0; /* List of indexed columns */
|
| + char **pz = &p->zErrmsg;
|
| + const char *zIdx = pIter->zIdx;
|
| + char *zLimit = 0;
|
| +
|
| + if( nOffset ){
|
| + zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset);
|
| + if( !zLimit ) p->rc = SQLITE_NOMEM;
|
| + }
|
| +
|
| + if( zIdx ){
|
| + const char *zTbl = pIter->zTbl;
|
| + char *zImposterCols = 0; /* Columns for imposter table */
|
| + char *zImposterPK = 0; /* Primary key declaration for imposter */
|
| + char *zWhere = 0; /* WHERE clause on PK columns */
|
| + char *zBind = 0;
|
| + int nBind = 0;
|
| +
|
| + assert( pIter->eType!=RBU_PK_VTAB );
|
| + zCollist = rbuObjIterGetIndexCols(
|
| + p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind
|
| + );
|
| + zBind = rbuObjIterGetBindlist(p, nBind);
|
| +
|
| + /* Create the imposter table used to write to this index. */
|
| + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1);
|
| + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum);
|
| + rbuMPrintfExec(p, p->dbMain,
|
| + "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID",
|
| + zTbl, zImposterCols, zImposterPK
|
| + );
|
| + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
|
| +
|
| + /* Create the statement to insert index entries */
|
| + pIter->nCol = nBind;
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = prepareFreeAndCollectError(
|
| + p->dbMain, &pIter->pInsert, &p->zErrmsg,
|
| + sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind)
|
| + );
|
| + }
|
| +
|
| + /* And to delete index entries */
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = prepareFreeAndCollectError(
|
| + p->dbMain, &pIter->pDelete, &p->zErrmsg,
|
| + sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere)
|
| + );
|
| + }
|
| +
|
| + /* Create the SELECT statement to read keys in sorted order */
|
| + if( p->rc==SQLITE_OK ){
|
| + char *zSql;
|
| + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
|
| + zSql = sqlite3_mprintf(
|
| + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s",
|
| + zCollist, p->zStateDb, pIter->zDataTbl,
|
| + zCollist, zLimit
|
| + );
|
| + }else{
|
| + zSql = sqlite3_mprintf(
|
| + "SELECT %s, rbu_control FROM '%q' "
|
| + "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 "
|
| + "UNION ALL "
|
| + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' "
|
| + "ORDER BY %s%s",
|
| + zCollist, pIter->zDataTbl,
|
| + zCollist, p->zStateDb, pIter->zDataTbl,
|
| + zCollist, zLimit
|
| + );
|
| + }
|
| + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql);
|
| + }
|
| +
|
| + sqlite3_free(zImposterCols);
|
| + sqlite3_free(zImposterPK);
|
| + sqlite3_free(zWhere);
|
| + sqlite3_free(zBind);
|
| + }else{
|
| + int bRbuRowid = (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE);
|
| + const char *zTbl = pIter->zTbl; /* Table this step applies to */
|
| + const char *zWrite; /* Imposter table name */
|
| +
|
| + char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid);
|
| + char *zWhere = rbuObjIterGetWhere(p, pIter);
|
| + char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old");
|
| + char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new");
|
| +
|
| + zCollist = rbuObjIterGetCollist(p, pIter);
|
| + pIter->nCol = pIter->nTblCol;
|
| +
|
| + /* Create the imposter table or tables (if required). */
|
| + rbuCreateImposterTable(p, pIter);
|
| + rbuCreateImposterTable2(p, pIter);
|
| + zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_");
|
| +
|
| + /* Create the INSERT statement to write to the target PK b-tree */
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz,
|
| + sqlite3_mprintf(
|
| + "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)",
|
| + zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings
|
| + )
|
| + );
|
| + }
|
| +
|
| + /* Create the DELETE statement to write to the target PK b-tree */
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz,
|
| + sqlite3_mprintf(
|
| + "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere
|
| + )
|
| + );
|
| + }
|
| +
|
| + if( pIter->abIndexed ){
|
| + const char *zRbuRowid = "";
|
| + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
|
| + zRbuRowid = ", rbu_rowid";
|
| + }
|
| +
|
| + /* Create the rbu_tmp_xxx table and the triggers to populate it. */
|
| + rbuMPrintfExec(p, p->dbRbu,
|
| + "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS "
|
| + "SELECT *%s FROM '%q' WHERE 0;"
|
| + , p->zStateDb, pIter->zDataTbl
|
| + , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "")
|
| + , pIter->zDataTbl
|
| + );
|
| +
|
| + rbuMPrintfExec(p, p->dbMain,
|
| + "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" "
|
| + "BEGIN "
|
| + " SELECT rbu_tmp_insert(2, %s);"
|
| + "END;"
|
| +
|
| + "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" "
|
| + "BEGIN "
|
| + " SELECT rbu_tmp_insert(2, %s);"
|
| + "END;"
|
| +
|
| + "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" "
|
| + "BEGIN "
|
| + " SELECT rbu_tmp_insert(3, %s);"
|
| + "END;",
|
| + zWrite, zTbl, zOldlist,
|
| + zWrite, zTbl, zOldlist,
|
| + zWrite, zTbl, zNewlist
|
| + );
|
| +
|
| + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
|
| + rbuMPrintfExec(p, p->dbMain,
|
| + "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" "
|
| + "BEGIN "
|
| + " SELECT rbu_tmp_insert(0, %s);"
|
| + "END;",
|
| + zWrite, zTbl, zNewlist
|
| + );
|
| + }
|
| +
|
| + rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid);
|
| + }
|
| +
|
| + /* Create the SELECT statement to read keys from data_xxx */
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz,
|
| + sqlite3_mprintf(
|
| + "SELECT %s, rbu_control%s FROM '%q'%s",
|
| + zCollist, (bRbuRowid ? ", rbu_rowid" : ""),
|
| + pIter->zDataTbl, zLimit
|
| + )
|
| + );
|
| + }
|
| +
|
| + sqlite3_free(zWhere);
|
| + sqlite3_free(zOldlist);
|
| + sqlite3_free(zNewlist);
|
| + sqlite3_free(zBindings);
|
| + }
|
| + sqlite3_free(zCollist);
|
| + sqlite3_free(zLimit);
|
| + }
|
| +
|
| + return p->rc;
|
| +}
|
| +
|
| +/*
|
| +** Set output variable *ppStmt to point to an UPDATE statement that may
|
| +** be used to update the imposter table for the main table b-tree of the
|
| +** table object that pIter currently points to, assuming that the
|
| +** rbu_control column of the data_xyz table contains zMask.
|
| +**
|
| +** If the zMask string does not specify any columns to update, then this
|
| +** is not an error. Output variable *ppStmt is set to NULL in this case.
|
| +*/
|
| +static int rbuGetUpdateStmt(
|
| + sqlite3rbu *p, /* RBU handle */
|
| + RbuObjIter *pIter, /* Object iterator */
|
| + const char *zMask, /* rbu_control value ('x.x.') */
|
| + sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */
|
| +){
|
| + RbuUpdateStmt **pp;
|
| + RbuUpdateStmt *pUp = 0;
|
| + int nUp = 0;
|
| +
|
| + /* In case an error occurs */
|
| + *ppStmt = 0;
|
| +
|
| + /* Search for an existing statement. If one is found, shift it to the front
|
| + ** of the LRU queue and return immediately. Otherwise, leave nUp pointing
|
| + ** to the number of statements currently in the cache and pUp to the
|
| + ** last object in the list. */
|
| + for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){
|
| + pUp = *pp;
|
| + if( strcmp(pUp->zMask, zMask)==0 ){
|
| + *pp = pUp->pNext;
|
| + pUp->pNext = pIter->pRbuUpdate;
|
| + pIter->pRbuUpdate = pUp;
|
| + *ppStmt = pUp->pUpdate;
|
| + return SQLITE_OK;
|
| + }
|
| + nUp++;
|
| + }
|
| + assert( pUp==0 || pUp->pNext==0 );
|
| +
|
| + if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){
|
| + for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext));
|
| + *pp = 0;
|
| + sqlite3_finalize(pUp->pUpdate);
|
| + pUp->pUpdate = 0;
|
| + }else{
|
| + pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1);
|
| + }
|
| +
|
| + if( pUp ){
|
| + char *zWhere = rbuObjIterGetWhere(p, pIter);
|
| + char *zSet = rbuObjIterGetSetlist(p, pIter, zMask);
|
| + char *zUpdate = 0;
|
| +
|
| + pUp->zMask = (char*)&pUp[1];
|
| + memcpy(pUp->zMask, zMask, pIter->nTblCol);
|
| + pUp->pNext = pIter->pRbuUpdate;
|
| + pIter->pRbuUpdate = pUp;
|
| +
|
| + if( zSet ){
|
| + const char *zPrefix = "";
|
| +
|
| + if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_";
|
| + zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s",
|
| + zPrefix, pIter->zTbl, zSet, zWhere
|
| + );
|
| + p->rc = prepareFreeAndCollectError(
|
| + p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate
|
| + );
|
| + *ppStmt = pUp->pUpdate;
|
| + }
|
| + sqlite3_free(zWhere);
|
| + sqlite3_free(zSet);
|
| + }
|
| +
|
| + return p->rc;
|
| +}
|
| +
|
| +static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){
|
| + sqlite3 *db = 0;
|
| + if( p->rc==SQLITE_OK ){
|
| + const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI;
|
| + p->rc = sqlite3_open_v2(zName, &db, flags, p->zVfsName);
|
| + if( p->rc ){
|
| + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
|
| + sqlite3_close(db);
|
| + db = 0;
|
| + }
|
| + }
|
| + return db;
|
| +}
|
| +
|
| +/*
|
| +** Open the database handle and attach the RBU database as "rbu". If an
|
| +** error occurs, leave an error code and message in the RBU handle.
|
| +*/
|
| +static void rbuOpenDatabase(sqlite3rbu *p){
|
| + assert( p->rc==SQLITE_OK );
|
| + assert( p->dbMain==0 && p->dbRbu==0 );
|
| +
|
| + p->eStage = 0;
|
| + p->dbMain = rbuOpenDbhandle(p, p->zTarget);
|
| + p->dbRbu = rbuOpenDbhandle(p, p->zRbu);
|
| +
|
| + /* If using separate RBU and state databases, attach the state database to
|
| + ** the RBU db handle now. */
|
| + if( p->zState ){
|
| + rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState);
|
| + memcpy(p->zStateDb, "stat", 4);
|
| + }else{
|
| + memcpy(p->zStateDb, "main", 4);
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = sqlite3_create_function(p->dbMain,
|
| + "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0
|
| + );
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = sqlite3_create_function(p->dbMain,
|
| + "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0
|
| + );
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = sqlite3_create_function(p->dbRbu,
|
| + "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0
|
| + );
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p);
|
| + }
|
| + rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master");
|
| +
|
| + /* Mark the database file just opened as an RBU target database. If
|
| + ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use.
|
| + ** This is an error. */
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p);
|
| + }
|
| +
|
| + if( p->rc==SQLITE_NOTFOUND ){
|
| + p->rc = SQLITE_ERROR;
|
| + p->zErrmsg = sqlite3_mprintf("rbu vfs not found");
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** This routine is a copy of the sqlite3FileSuffix3() routine from the core.
|
| +** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined.
|
| +**
|
| +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database
|
| +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and
|
| +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than
|
| +** three characters, then shorten the suffix on z[] to be the last three
|
| +** characters of the original suffix.
|
| +**
|
| +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always
|
| +** do the suffix shortening regardless of URI parameter.
|
| +**
|
| +** Examples:
|
| +**
|
| +** test.db-journal => test.nal
|
| +** test.db-wal => test.wal
|
| +** test.db-shm => test.shm
|
| +** test.db-mj7f3319fa => test.9fa
|
| +*/
|
| +static void rbuFileSuffix3(const char *zBase, char *z){
|
| +#ifdef SQLITE_ENABLE_8_3_NAMES
|
| +#if SQLITE_ENABLE_8_3_NAMES<2
|
| + if( sqlite3_uri_boolean(zBase, "8_3_names", 0) )
|
| +#endif
|
| + {
|
| + int i, sz;
|
| + sz = sqlite3Strlen30(z);
|
| + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
|
| + if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4);
|
| + }
|
| +#endif
|
| +}
|
| +
|
| +/*
|
| +** Return the current wal-index header checksum for the target database
|
| +** as a 64-bit integer.
|
| +**
|
| +** The checksum is store in the first page of xShmMap memory as an 8-byte
|
| +** blob starting at byte offset 40.
|
| +*/
|
| +static i64 rbuShmChecksum(sqlite3rbu *p){
|
| + i64 iRet = 0;
|
| + if( p->rc==SQLITE_OK ){
|
| + sqlite3_file *pDb = p->pTargetFd->pReal;
|
| + u32 volatile *ptr;
|
| + p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr);
|
| + if( p->rc==SQLITE_OK ){
|
| + iRet = ((i64)ptr[10] << 32) + ptr[11];
|
| + }
|
| + }
|
| + return iRet;
|
| +}
|
| +
|
| +/*
|
| +** This function is called as part of initializing or reinitializing an
|
| +** incremental checkpoint.
|
| +**
|
| +** It populates the sqlite3rbu.aFrame[] array with the set of
|
| +** (wal frame -> db page) copy operations required to checkpoint the
|
| +** current wal file, and obtains the set of shm locks required to safely
|
| +** perform the copy operations directly on the file-system.
|
| +**
|
| +** If argument pState is not NULL, then the incremental checkpoint is
|
| +** being resumed. In this case, if the checksum of the wal-index-header
|
| +** following recovery is not the same as the checksum saved in the RbuState
|
| +** object, then the rbu handle is set to DONE state. This occurs if some
|
| +** other client appends a transaction to the wal file in the middle of
|
| +** an incremental checkpoint.
|
| +*/
|
| +static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){
|
| +
|
| + /* If pState is NULL, then the wal file may not have been opened and
|
| + ** recovered. Running a read-statement here to ensure that doing so
|
| + ** does not interfere with the "capture" process below. */
|
| + if( pState==0 ){
|
| + p->eStage = 0;
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_master", 0, 0, 0);
|
| + }
|
| + }
|
| +
|
| + /* Assuming no error has occurred, run a "restart" checkpoint with the
|
| + ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following
|
| + ** special behaviour in the rbu VFS:
|
| + **
|
| + ** * If the exclusive shm WRITER or READ0 lock cannot be obtained,
|
| + ** the checkpoint fails with SQLITE_BUSY (normally SQLite would
|
| + ** proceed with running a passive checkpoint instead of failing).
|
| + **
|
| + ** * Attempts to read from the *-wal file or write to the database file
|
| + ** do not perform any IO. Instead, the frame/page combinations that
|
| + ** would be read/written are recorded in the sqlite3rbu.aFrame[]
|
| + ** array.
|
| + **
|
| + ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER,
|
| + ** READ0 and CHECKPOINT locks taken as part of the checkpoint are
|
| + ** no-ops. These locks will not be released until the connection
|
| + ** is closed.
|
| + **
|
| + ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL
|
| + ** error.
|
| + **
|
| + ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the
|
| + ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[]
|
| + ** array populated with a set of (frame -> page) mappings. Because the
|
| + ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy
|
| + ** data from the wal file into the database file according to the
|
| + ** contents of aFrame[].
|
| + */
|
| + if( p->rc==SQLITE_OK ){
|
| + int rc2;
|
| + p->eStage = RBU_STAGE_CAPTURE;
|
| + rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0);
|
| + if( rc2!=SQLITE_INTERNAL ) p->rc = rc2;
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + p->eStage = RBU_STAGE_CKPT;
|
| + p->nStep = (pState ? pState->nRow : 0);
|
| + p->aBuf = rbuMalloc(p, p->pgsz);
|
| + p->iWalCksum = rbuShmChecksum(p);
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){
|
| + p->rc = SQLITE_DONE;
|
| + p->eStage = RBU_STAGE_DONE;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Called when iAmt bytes are read from offset iOff of the wal file while
|
| +** the rbu object is in capture mode. Record the frame number of the frame
|
| +** being read in the aFrame[] array.
|
| +*/
|
| +static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){
|
| + const u32 mReq = (1<<WAL_LOCK_WRITE)|(1<<WAL_LOCK_CKPT)|(1<<WAL_LOCK_READ0);
|
| + u32 iFrame;
|
| +
|
| + if( pRbu->mLock!=mReq ){
|
| + pRbu->rc = SQLITE_BUSY;
|
| + return SQLITE_INTERNAL;
|
| + }
|
| +
|
| + pRbu->pgsz = iAmt;
|
| + if( pRbu->nFrame==pRbu->nFrameAlloc ){
|
| + int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2;
|
| + RbuFrame *aNew;
|
| + aNew = (RbuFrame*)sqlite3_realloc(pRbu->aFrame, nNew * sizeof(RbuFrame));
|
| + if( aNew==0 ) return SQLITE_NOMEM;
|
| + pRbu->aFrame = aNew;
|
| + pRbu->nFrameAlloc = nNew;
|
| + }
|
| +
|
| + iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1;
|
| + if( pRbu->iMaxFrame<iFrame ) pRbu->iMaxFrame = iFrame;
|
| + pRbu->aFrame[pRbu->nFrame].iWalFrame = iFrame;
|
| + pRbu->aFrame[pRbu->nFrame].iDbPage = 0;
|
| + pRbu->nFrame++;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Called when a page of data is written to offset iOff of the database
|
| +** file while the rbu handle is in capture mode. Record the page number
|
| +** of the page being written in the aFrame[] array.
|
| +*/
|
| +static int rbuCaptureDbWrite(sqlite3rbu *pRbu, i64 iOff){
|
| + pRbu->aFrame[pRbu->nFrame-1].iDbPage = (u32)(iOff / pRbu->pgsz) + 1;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** This is called as part of an incremental checkpoint operation. Copy
|
| +** a single frame of data from the wal file into the database file, as
|
| +** indicated by the RbuFrame object.
|
| +*/
|
| +static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){
|
| + sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal;
|
| + sqlite3_file *pDb = p->pTargetFd->pReal;
|
| + i64 iOff;
|
| +
|
| + assert( p->rc==SQLITE_OK );
|
| + iOff = (i64)(pFrame->iWalFrame-1) * (p->pgsz + 24) + 32 + 24;
|
| + p->rc = pWal->pMethods->xRead(pWal, p->aBuf, p->pgsz, iOff);
|
| + if( p->rc ) return;
|
| +
|
| + iOff = (i64)(pFrame->iDbPage-1) * p->pgsz;
|
| + p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff);
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Take an EXCLUSIVE lock on the database file.
|
| +*/
|
| +static void rbuLockDatabase(sqlite3rbu *p){
|
| + sqlite3_file *pReal = p->pTargetFd->pReal;
|
| + assert( p->rc==SQLITE_OK );
|
| + p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_SHARED);
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_EXCLUSIVE);
|
| + }
|
| +}
|
| +
|
| +#if defined(_WIN32_WCE)
|
| +static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){
|
| + int nChar;
|
| + LPWSTR zWideFilename;
|
| +
|
| + nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0);
|
| + if( nChar==0 ){
|
| + return 0;
|
| + }
|
| + zWideFilename = sqlite3_malloc( nChar*sizeof(zWideFilename[0]) );
|
| + if( zWideFilename==0 ){
|
| + return 0;
|
| + }
|
| + memset(zWideFilename, 0, nChar*sizeof(zWideFilename[0]));
|
| + nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename,
|
| + nChar);
|
| + if( nChar==0 ){
|
| + sqlite3_free(zWideFilename);
|
| + zWideFilename = 0;
|
| + }
|
| + return zWideFilename;
|
| +}
|
| +#endif
|
| +
|
| +/*
|
| +** The RBU handle is currently in RBU_STAGE_OAL state, with a SHARED lock
|
| +** on the database file. This proc moves the *-oal file to the *-wal path,
|
| +** then reopens the database file (this time in vanilla, non-oal, WAL mode).
|
| +** If an error occurs, leave an error code and error message in the rbu
|
| +** handle.
|
| +*/
|
| +static void rbuMoveOalFile(sqlite3rbu *p){
|
| + const char *zBase = sqlite3_db_filename(p->dbMain, "main");
|
| +
|
| + char *zWal = sqlite3_mprintf("%s-wal", zBase);
|
| + char *zOal = sqlite3_mprintf("%s-oal", zBase);
|
| +
|
| + assert( p->eStage==RBU_STAGE_MOVE );
|
| + assert( p->rc==SQLITE_OK && p->zErrmsg==0 );
|
| + if( zWal==0 || zOal==0 ){
|
| + p->rc = SQLITE_NOMEM;
|
| + }else{
|
| + /* Move the *-oal file to *-wal. At this point connection p->db is
|
| + ** holding a SHARED lock on the target database file (because it is
|
| + ** in WAL mode). So no other connection may be writing the db.
|
| + **
|
| + ** In order to ensure that there are no database readers, an EXCLUSIVE
|
| + ** lock is obtained here before the *-oal is moved to *-wal.
|
| + */
|
| + rbuLockDatabase(p);
|
| + if( p->rc==SQLITE_OK ){
|
| + rbuFileSuffix3(zBase, zWal);
|
| + rbuFileSuffix3(zBase, zOal);
|
| +
|
| + /* Re-open the databases. */
|
| + rbuObjIterFinalize(&p->objiter);
|
| + sqlite3_close(p->dbMain);
|
| + sqlite3_close(p->dbRbu);
|
| + p->dbMain = 0;
|
| + p->dbRbu = 0;
|
| +
|
| +#if defined(_WIN32_WCE)
|
| + {
|
| + LPWSTR zWideOal;
|
| + LPWSTR zWideWal;
|
| +
|
| + zWideOal = rbuWinUtf8ToUnicode(zOal);
|
| + if( zWideOal ){
|
| + zWideWal = rbuWinUtf8ToUnicode(zWal);
|
| + if( zWideWal ){
|
| + if( MoveFileW(zWideOal, zWideWal) ){
|
| + p->rc = SQLITE_OK;
|
| + }else{
|
| + p->rc = SQLITE_IOERR;
|
| + }
|
| + sqlite3_free(zWideWal);
|
| + }else{
|
| + p->rc = SQLITE_IOERR_NOMEM;
|
| + }
|
| + sqlite3_free(zWideOal);
|
| + }else{
|
| + p->rc = SQLITE_IOERR_NOMEM;
|
| + }
|
| + }
|
| +#else
|
| + p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK;
|
| +#endif
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + rbuOpenDatabase(p);
|
| + rbuSetupCheckpoint(p, 0);
|
| + }
|
| + }
|
| + }
|
| +
|
| + sqlite3_free(zWal);
|
| + sqlite3_free(zOal);
|
| +}
|
| +
|
| +/*
|
| +** The SELECT statement iterating through the keys for the current object
|
| +** (p->objiter.pSelect) currently points to a valid row. This function
|
| +** determines the type of operation requested by this row and returns
|
| +** one of the following values to indicate the result:
|
| +**
|
| +** * RBU_INSERT
|
| +** * RBU_DELETE
|
| +** * RBU_IDX_DELETE
|
| +** * RBU_UPDATE
|
| +**
|
| +** If RBU_UPDATE is returned, then output variable *pzMask is set to
|
| +** point to the text value indicating the columns to update.
|
| +**
|
| +** If the rbu_control field contains an invalid value, an error code and
|
| +** message are left in the RBU handle and zero returned.
|
| +*/
|
| +static int rbuStepType(sqlite3rbu *p, const char **pzMask){
|
| + int iCol = p->objiter.nCol; /* Index of rbu_control column */
|
| + int res = 0; /* Return value */
|
| +
|
| + switch( sqlite3_column_type(p->objiter.pSelect, iCol) ){
|
| + case SQLITE_INTEGER: {
|
| + int iVal = sqlite3_column_int(p->objiter.pSelect, iCol);
|
| + if( iVal==0 ){
|
| + res = RBU_INSERT;
|
| + }else if( iVal==1 ){
|
| + res = RBU_DELETE;
|
| + }else if( iVal==2 ){
|
| + res = RBU_IDX_DELETE;
|
| + }else if( iVal==3 ){
|
| + res = RBU_IDX_INSERT;
|
| + }
|
| + break;
|
| + }
|
| +
|
| + case SQLITE_TEXT: {
|
| + const unsigned char *z = sqlite3_column_text(p->objiter.pSelect, iCol);
|
| + if( z==0 ){
|
| + p->rc = SQLITE_NOMEM;
|
| + }else{
|
| + *pzMask = (const char*)z;
|
| + }
|
| + res = RBU_UPDATE;
|
| +
|
| + break;
|
| + }
|
| +
|
| + default:
|
| + break;
|
| + }
|
| +
|
| + if( res==0 ){
|
| + rbuBadControlError(p);
|
| + }
|
| + return res;
|
| +}
|
| +
|
| +#ifdef SQLITE_DEBUG
|
| +/*
|
| +** Assert that column iCol of statement pStmt is named zName.
|
| +*/
|
| +static void assertColumnName(sqlite3_stmt *pStmt, int iCol, const char *zName){
|
| + const char *zCol = sqlite3_column_name(pStmt, iCol);
|
| + assert( 0==sqlite3_stricmp(zName, zCol) );
|
| +}
|
| +#else
|
| +# define assertColumnName(x,y,z)
|
| +#endif
|
| +
|
| +/*
|
| +** This function does the work for an sqlite3rbu_step() call.
|
| +**
|
| +** The object-iterator (p->objiter) currently points to a valid object,
|
| +** and the input cursor (p->objiter.pSelect) currently points to a valid
|
| +** input row. Perform whatever processing is required and return.
|
| +**
|
| +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code
|
| +** and message is left in the RBU handle and a copy of the error code
|
| +** returned.
|
| +*/
|
| +static int rbuStep(sqlite3rbu *p){
|
| + RbuObjIter *pIter = &p->objiter;
|
| + const char *zMask = 0;
|
| + int i;
|
| + int eType = rbuStepType(p, &zMask);
|
| +
|
| + if( eType ){
|
| + assert( eType!=RBU_UPDATE || pIter->zIdx==0 );
|
| +
|
| + if( pIter->zIdx==0 && eType==RBU_IDX_DELETE ){
|
| + rbuBadControlError(p);
|
| + }
|
| + else if(
|
| + eType==RBU_INSERT
|
| + || eType==RBU_DELETE
|
| + || eType==RBU_IDX_DELETE
|
| + || eType==RBU_IDX_INSERT
|
| + ){
|
| + sqlite3_value *pVal;
|
| + sqlite3_stmt *pWriter;
|
| +
|
| + assert( eType!=RBU_UPDATE );
|
| + assert( eType!=RBU_DELETE || pIter->zIdx==0 );
|
| +
|
| + if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){
|
| + pWriter = pIter->pDelete;
|
| + }else{
|
| + pWriter = pIter->pInsert;
|
| + }
|
| +
|
| + for(i=0; i<pIter->nCol; i++){
|
| + /* If this is an INSERT into a table b-tree and the table has an
|
| + ** explicit INTEGER PRIMARY KEY, check that this is not an attempt
|
| + ** to write a NULL into the IPK column. That is not permitted. */
|
| + if( eType==RBU_INSERT
|
| + && pIter->zIdx==0 && pIter->eType==RBU_PK_IPK && pIter->abTblPk[i]
|
| + && sqlite3_column_type(pIter->pSelect, i)==SQLITE_NULL
|
| + ){
|
| + p->rc = SQLITE_MISMATCH;
|
| + p->zErrmsg = sqlite3_mprintf("datatype mismatch");
|
| + goto step_out;
|
| + }
|
| +
|
| + if( eType==RBU_DELETE && pIter->abTblPk[i]==0 ){
|
| + continue;
|
| + }
|
| +
|
| + pVal = sqlite3_column_value(pIter->pSelect, i);
|
| + p->rc = sqlite3_bind_value(pWriter, i+1, pVal);
|
| + if( p->rc ) goto step_out;
|
| + }
|
| + if( pIter->zIdx==0
|
| + && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE)
|
| + ){
|
| + /* For a virtual table, or a table with no primary key, the
|
| + ** SELECT statement is:
|
| + **
|
| + ** SELECT <cols>, rbu_control, rbu_rowid FROM ....
|
| + **
|
| + ** Hence column_value(pIter->nCol+1).
|
| + */
|
| + assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid");
|
| + pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1);
|
| + p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal);
|
| + }
|
| + if( p->rc==SQLITE_OK ){
|
| + sqlite3_step(pWriter);
|
| + p->rc = resetAndCollectError(pWriter, &p->zErrmsg);
|
| + }
|
| + }else{
|
| + sqlite3_value *pVal;
|
| + sqlite3_stmt *pUpdate = 0;
|
| + assert( eType==RBU_UPDATE );
|
| + rbuGetUpdateStmt(p, pIter, zMask, &pUpdate);
|
| + if( pUpdate ){
|
| + for(i=0; p->rc==SQLITE_OK && i<pIter->nCol; i++){
|
| + char c = zMask[pIter->aiSrcOrder[i]];
|
| + pVal = sqlite3_column_value(pIter->pSelect, i);
|
| + if( pIter->abTblPk[i] || c!='.' ){
|
| + p->rc = sqlite3_bind_value(pUpdate, i+1, pVal);
|
| + }
|
| + }
|
| + if( p->rc==SQLITE_OK
|
| + && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE)
|
| + ){
|
| + /* Bind the rbu_rowid value to column _rowid_ */
|
| + assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid");
|
| + pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1);
|
| + p->rc = sqlite3_bind_value(pUpdate, pIter->nCol+1, pVal);
|
| + }
|
| + if( p->rc==SQLITE_OK ){
|
| + sqlite3_step(pUpdate);
|
| + p->rc = resetAndCollectError(pUpdate, &p->zErrmsg);
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + step_out:
|
| + return p->rc;
|
| +}
|
| +
|
| +/*
|
| +** Increment the schema cookie of the main database opened by p->dbMain.
|
| +*/
|
| +static void rbuIncrSchemaCookie(sqlite3rbu *p){
|
| + if( p->rc==SQLITE_OK ){
|
| + int iCookie = 1000000;
|
| + sqlite3_stmt *pStmt;
|
| +
|
| + p->rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg,
|
| + "PRAGMA schema_version"
|
| + );
|
| + if( p->rc==SQLITE_OK ){
|
| + /* Coverage: it may be that this sqlite3_step() cannot fail. There
|
| + ** is already a transaction open, so the prepared statement cannot
|
| + ** throw an SQLITE_SCHEMA exception. The only database page the
|
| + ** statement reads is page 1, which is guaranteed to be in the cache.
|
| + ** And no memory allocations are required. */
|
| + if( SQLITE_ROW==sqlite3_step(pStmt) ){
|
| + iCookie = sqlite3_column_int(pStmt, 0);
|
| + }
|
| + rbuFinalize(p, pStmt);
|
| + }
|
| + if( p->rc==SQLITE_OK ){
|
| + rbuMPrintfExec(p, p->dbMain, "PRAGMA schema_version = %d", iCookie+1);
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Update the contents of the rbu_state table within the rbu database. The
|
| +** value stored in the RBU_STATE_STAGE column is eStage. All other values
|
| +** are determined by inspecting the rbu handle passed as the first argument.
|
| +*/
|
| +static void rbuSaveState(sqlite3rbu *p, int eStage){
|
| + if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){
|
| + sqlite3_stmt *pInsert = 0;
|
| + int rc;
|
| +
|
| + assert( p->zErrmsg==0 );
|
| + rc = prepareFreeAndCollectError(p->dbRbu, &pInsert, &p->zErrmsg,
|
| + sqlite3_mprintf(
|
| + "INSERT OR REPLACE INTO %s.rbu_state(k, v) VALUES "
|
| + "(%d, %d), "
|
| + "(%d, %Q), "
|
| + "(%d, %Q), "
|
| + "(%d, %d), "
|
| + "(%d, %d), "
|
| + "(%d, %lld), "
|
| + "(%d, %lld), "
|
| + "(%d, %lld) ",
|
| + p->zStateDb,
|
| + RBU_STATE_STAGE, eStage,
|
| + RBU_STATE_TBL, p->objiter.zTbl,
|
| + RBU_STATE_IDX, p->objiter.zIdx,
|
| + RBU_STATE_ROW, p->nStep,
|
| + RBU_STATE_PROGRESS, p->nProgress,
|
| + RBU_STATE_CKPT, p->iWalCksum,
|
| + RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie,
|
| + RBU_STATE_OALSZ, p->iOalSz
|
| + )
|
| + );
|
| + assert( pInsert==0 || rc==SQLITE_OK );
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_step(pInsert);
|
| + rc = sqlite3_finalize(pInsert);
|
| + }
|
| + if( rc!=SQLITE_OK ) p->rc = rc;
|
| + }
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Step the RBU object.
|
| +*/
|
| +SQLITE_API int SQLITE_STDCALL sqlite3rbu_step(sqlite3rbu *p){
|
| + if( p ){
|
| + switch( p->eStage ){
|
| + case RBU_STAGE_OAL: {
|
| + RbuObjIter *pIter = &p->objiter;
|
| + while( p->rc==SQLITE_OK && pIter->zTbl ){
|
| +
|
| + if( pIter->bCleanup ){
|
| + /* Clean up the rbu_tmp_xxx table for the previous table. It
|
| + ** cannot be dropped as there are currently active SQL statements.
|
| + ** But the contents can be deleted. */
|
| + if( pIter->abIndexed ){
|
| + rbuMPrintfExec(p, p->dbRbu,
|
| + "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl
|
| + );
|
| + }
|
| + }else{
|
| + rbuObjIterPrepareAll(p, pIter, 0);
|
| +
|
| + /* Advance to the next row to process. */
|
| + if( p->rc==SQLITE_OK ){
|
| + int rc = sqlite3_step(pIter->pSelect);
|
| + if( rc==SQLITE_ROW ){
|
| + p->nProgress++;
|
| + p->nStep++;
|
| + return rbuStep(p);
|
| + }
|
| + p->rc = sqlite3_reset(pIter->pSelect);
|
| + p->nStep = 0;
|
| + }
|
| + }
|
| +
|
| + rbuObjIterNext(p, pIter);
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + assert( pIter->zTbl==0 );
|
| + rbuSaveState(p, RBU_STAGE_MOVE);
|
| + rbuIncrSchemaCookie(p);
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg);
|
| + }
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
|
| + }
|
| + p->eStage = RBU_STAGE_MOVE;
|
| + }
|
| + break;
|
| + }
|
| +
|
| + case RBU_STAGE_MOVE: {
|
| + if( p->rc==SQLITE_OK ){
|
| + rbuMoveOalFile(p);
|
| + p->nProgress++;
|
| + }
|
| + break;
|
| + }
|
| +
|
| + case RBU_STAGE_CKPT: {
|
| + if( p->rc==SQLITE_OK ){
|
| + if( p->nStep>=p->nFrame ){
|
| + sqlite3_file *pDb = p->pTargetFd->pReal;
|
| +
|
| + /* Sync the db file */
|
| + p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
|
| +
|
| + /* Update nBackfill */
|
| + if( p->rc==SQLITE_OK ){
|
| + void volatile *ptr;
|
| + p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, &ptr);
|
| + if( p->rc==SQLITE_OK ){
|
| + ((u32 volatile*)ptr)[24] = p->iMaxFrame;
|
| + }
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + p->eStage = RBU_STAGE_DONE;
|
| + p->rc = SQLITE_DONE;
|
| + }
|
| + }else{
|
| + RbuFrame *pFrame = &p->aFrame[p->nStep];
|
| + rbuCheckpointFrame(p, pFrame);
|
| + p->nStep++;
|
| + }
|
| + p->nProgress++;
|
| + }
|
| + break;
|
| + }
|
| +
|
| + default:
|
| + break;
|
| + }
|
| + return p->rc;
|
| + }else{
|
| + return SQLITE_NOMEM;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Free an RbuState object allocated by rbuLoadState().
|
| +*/
|
| +static void rbuFreeState(RbuState *p){
|
| + if( p ){
|
| + sqlite3_free(p->zTbl);
|
| + sqlite3_free(p->zIdx);
|
| + sqlite3_free(p);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Allocate an RbuState object and load the contents of the rbu_state
|
| +** table into it. Return a pointer to the new object. It is the
|
| +** responsibility of the caller to eventually free the object using
|
| +** sqlite3_free().
|
| +**
|
| +** If an error occurs, leave an error code and message in the rbu handle
|
| +** and return NULL.
|
| +*/
|
| +static RbuState *rbuLoadState(sqlite3rbu *p){
|
| + RbuState *pRet = 0;
|
| + sqlite3_stmt *pStmt = 0;
|
| + int rc;
|
| + int rc2;
|
| +
|
| + pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState));
|
| + if( pRet==0 ) return 0;
|
| +
|
| + rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
|
| + sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb)
|
| + );
|
| + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
|
| + switch( sqlite3_column_int(pStmt, 0) ){
|
| + case RBU_STATE_STAGE:
|
| + pRet->eStage = sqlite3_column_int(pStmt, 1);
|
| + if( pRet->eStage!=RBU_STAGE_OAL
|
| + && pRet->eStage!=RBU_STAGE_MOVE
|
| + && pRet->eStage!=RBU_STAGE_CKPT
|
| + ){
|
| + p->rc = SQLITE_CORRUPT;
|
| + }
|
| + break;
|
| +
|
| + case RBU_STATE_TBL:
|
| + pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
|
| + break;
|
| +
|
| + case RBU_STATE_IDX:
|
| + pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
|
| + break;
|
| +
|
| + case RBU_STATE_ROW:
|
| + pRet->nRow = sqlite3_column_int(pStmt, 1);
|
| + break;
|
| +
|
| + case RBU_STATE_PROGRESS:
|
| + pRet->nProgress = sqlite3_column_int64(pStmt, 1);
|
| + break;
|
| +
|
| + case RBU_STATE_CKPT:
|
| + pRet->iWalCksum = sqlite3_column_int64(pStmt, 1);
|
| + break;
|
| +
|
| + case RBU_STATE_COOKIE:
|
| + pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1);
|
| + break;
|
| +
|
| + case RBU_STATE_OALSZ:
|
| + pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1);
|
| + break;
|
| +
|
| + default:
|
| + rc = SQLITE_CORRUPT;
|
| + break;
|
| + }
|
| + }
|
| + rc2 = sqlite3_finalize(pStmt);
|
| + if( rc==SQLITE_OK ) rc = rc2;
|
| +
|
| + p->rc = rc;
|
| + return pRet;
|
| +}
|
| +
|
| +/*
|
| +** Compare strings z1 and z2, returning 0 if they are identical, or non-zero
|
| +** otherwise. Either or both argument may be NULL. Two NULL values are
|
| +** considered equal, and NULL is considered distinct from all other values.
|
| +*/
|
| +static int rbuStrCompare(const char *z1, const char *z2){
|
| + if( z1==0 && z2==0 ) return 0;
|
| + if( z1==0 || z2==0 ) return 1;
|
| + return (sqlite3_stricmp(z1, z2)!=0);
|
| +}
|
| +
|
| +/*
|
| +** This function is called as part of sqlite3rbu_open() when initializing
|
| +** an rbu handle in OAL stage. If the rbu update has not started (i.e.
|
| +** the rbu_state table was empty) it is a no-op. Otherwise, it arranges
|
| +** things so that the next call to sqlite3rbu_step() continues on from
|
| +** where the previous rbu handle left off.
|
| +**
|
| +** If an error occurs, an error code and error message are left in the
|
| +** rbu handle passed as the first argument.
|
| +*/
|
| +static void rbuSetupOal(sqlite3rbu *p, RbuState *pState){
|
| + assert( p->rc==SQLITE_OK );
|
| + if( pState->zTbl ){
|
| + RbuObjIter *pIter = &p->objiter;
|
| + int rc = SQLITE_OK;
|
| +
|
| + while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup
|
| + || rbuStrCompare(pIter->zIdx, pState->zIdx)
|
| + || rbuStrCompare(pIter->zTbl, pState->zTbl)
|
| + )){
|
| + rc = rbuObjIterNext(p, pIter);
|
| + }
|
| +
|
| + if( rc==SQLITE_OK && !pIter->zTbl ){
|
| + rc = SQLITE_ERROR;
|
| + p->zErrmsg = sqlite3_mprintf("rbu_state mismatch error");
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + p->nStep = pState->nRow;
|
| + rc = rbuObjIterPrepareAll(p, &p->objiter, p->nStep);
|
| + }
|
| +
|
| + p->rc = rc;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** If there is a "*-oal" file in the file-system corresponding to the
|
| +** target database in the file-system, delete it. If an error occurs,
|
| +** leave an error code and error message in the rbu handle.
|
| +*/
|
| +static void rbuDeleteOalFile(sqlite3rbu *p){
|
| + char *zOal = rbuMPrintf(p, "%s-oal", p->zTarget);
|
| + if( zOal ){
|
| + sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
|
| + assert( pVfs && p->rc==SQLITE_OK && p->zErrmsg==0 );
|
| + pVfs->xDelete(pVfs, zOal, 0);
|
| + sqlite3_free(zOal);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Allocate a private rbu VFS for the rbu handle passed as the only
|
| +** argument. This VFS will be used unless the call to sqlite3rbu_open()
|
| +** specified a URI with a vfs=? option in place of a target database
|
| +** file name.
|
| +*/
|
| +static void rbuCreateVfs(sqlite3rbu *p){
|
| + int rnd;
|
| + char zRnd[64];
|
| +
|
| + assert( p->rc==SQLITE_OK );
|
| + sqlite3_randomness(sizeof(int), (void*)&rnd);
|
| + sqlite3_snprintf(sizeof(zRnd), zRnd, "rbu_vfs_%d", rnd);
|
| + p->rc = sqlite3rbu_create_vfs(zRnd, 0);
|
| + if( p->rc==SQLITE_OK ){
|
| + sqlite3_vfs *pVfs = sqlite3_vfs_find(zRnd);
|
| + assert( pVfs );
|
| + p->zVfsName = pVfs->zName;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Destroy the private VFS created for the rbu handle passed as the only
|
| +** argument by an earlier call to rbuCreateVfs().
|
| +*/
|
| +static void rbuDeleteVfs(sqlite3rbu *p){
|
| + if( p->zVfsName ){
|
| + sqlite3rbu_destroy_vfs(p->zVfsName);
|
| + p->zVfsName = 0;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Open and return a new RBU handle.
|
| +*/
|
| +SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open(
|
| + const char *zTarget,
|
| + const char *zRbu,
|
| + const char *zState
|
| +){
|
| + sqlite3rbu *p;
|
| + int nTarget = strlen(zTarget);
|
| + int nRbu = strlen(zRbu);
|
| + int nState = zState ? strlen(zState) : 0;
|
| +
|
| + p = (sqlite3rbu*)sqlite3_malloc(sizeof(sqlite3rbu)+nTarget+1+nRbu+1+nState+1);
|
| + if( p ){
|
| + RbuState *pState = 0;
|
| +
|
| + /* Create the custom VFS. */
|
| + memset(p, 0, sizeof(sqlite3rbu));
|
| + rbuCreateVfs(p);
|
| +
|
| + /* Open the target database */
|
| + if( p->rc==SQLITE_OK ){
|
| + p->zTarget = (char*)&p[1];
|
| + memcpy(p->zTarget, zTarget, nTarget+1);
|
| + p->zRbu = &p->zTarget[nTarget+1];
|
| + memcpy(p->zRbu, zRbu, nRbu+1);
|
| + if( zState ){
|
| + p->zState = &p->zRbu[nRbu+1];
|
| + memcpy(p->zState, zState, nState+1);
|
| + }
|
| + rbuOpenDatabase(p);
|
| + }
|
| +
|
| + /* If it has not already been created, create the rbu_state table */
|
| + rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb);
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + pState = rbuLoadState(p);
|
| + assert( pState || p->rc!=SQLITE_OK );
|
| + if( p->rc==SQLITE_OK ){
|
| +
|
| + if( pState->eStage==0 ){
|
| + rbuDeleteOalFile(p);
|
| + p->eStage = RBU_STAGE_OAL;
|
| + }else{
|
| + p->eStage = pState->eStage;
|
| + }
|
| + p->nProgress = pState->nProgress;
|
| + p->iOalSz = pState->iOalSz;
|
| + }
|
| + }
|
| + assert( p->rc!=SQLITE_OK || p->eStage!=0 );
|
| +
|
| + if( p->rc==SQLITE_OK && p->pTargetFd->pWalFd ){
|
| + if( p->eStage==RBU_STAGE_OAL ){
|
| + p->rc = SQLITE_ERROR;
|
| + p->zErrmsg = sqlite3_mprintf("cannot update wal mode database");
|
| + }else if( p->eStage==RBU_STAGE_MOVE ){
|
| + p->eStage = RBU_STAGE_CKPT;
|
| + p->nStep = 0;
|
| + }
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK
|
| + && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE)
|
| + && pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie
|
| + ){
|
| + /* At this point (pTargetFd->iCookie) contains the value of the
|
| + ** change-counter cookie (the thing that gets incremented when a
|
| + ** transaction is committed in rollback mode) currently stored on
|
| + ** page 1 of the database file. */
|
| + p->rc = SQLITE_BUSY;
|
| + p->zErrmsg = sqlite3_mprintf("database modified during rbu update");
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + if( p->eStage==RBU_STAGE_OAL ){
|
| + sqlite3 *db = p->dbMain;
|
| +
|
| + /* Open transactions both databases. The *-oal file is opened or
|
| + ** created at this point. */
|
| + p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
|
| + }
|
| +
|
| + /* Check if the main database is a zipvfs db. If it is, set the upper
|
| + ** level pager to use "journal_mode=off". This prevents it from
|
| + ** generating a large journal using a temp file. */
|
| + if( p->rc==SQLITE_OK ){
|
| + int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0);
|
| + if( frc==SQLITE_OK ){
|
| + p->rc = sqlite3_exec(db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg);
|
| + }
|
| + }
|
| +
|
| + /* Point the object iterator at the first object */
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = rbuObjIterFirst(p, &p->objiter);
|
| + }
|
| +
|
| + /* If the RBU database contains no data_xxx tables, declare the RBU
|
| + ** update finished. */
|
| + if( p->rc==SQLITE_OK && p->objiter.zTbl==0 ){
|
| + p->rc = SQLITE_DONE;
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + rbuSetupOal(p, pState);
|
| + }
|
| +
|
| + }else if( p->eStage==RBU_STAGE_MOVE ){
|
| + /* no-op */
|
| + }else if( p->eStage==RBU_STAGE_CKPT ){
|
| + rbuSetupCheckpoint(p, pState);
|
| + }else if( p->eStage==RBU_STAGE_DONE ){
|
| + p->rc = SQLITE_DONE;
|
| + }else{
|
| + p->rc = SQLITE_CORRUPT;
|
| + }
|
| + }
|
| +
|
| + rbuFreeState(pState);
|
| + }
|
| +
|
| + return p;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Return the database handle used by pRbu.
|
| +*/
|
| +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){
|
| + sqlite3 *db = 0;
|
| + if( pRbu ){
|
| + db = (bRbu ? pRbu->dbRbu : pRbu->dbMain);
|
| + }
|
| + return db;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** If the error code currently stored in the RBU handle is SQLITE_CONSTRAINT,
|
| +** then edit any error message string so as to remove all occurrences of
|
| +** the pattern "rbu_imp_[0-9]*".
|
| +*/
|
| +static void rbuEditErrmsg(sqlite3rbu *p){
|
| + if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){
|
| + int i;
|
| + int nErrmsg = strlen(p->zErrmsg);
|
| + for(i=0; i<(nErrmsg-8); i++){
|
| + if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){
|
| + int nDel = 8;
|
| + while( p->zErrmsg[i+nDel]>='0' && p->zErrmsg[i+nDel]<='9' ) nDel++;
|
| + memmove(&p->zErrmsg[i], &p->zErrmsg[i+nDel], nErrmsg + 1 - i - nDel);
|
| + nErrmsg -= nDel;
|
| + }
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Close the RBU handle.
|
| +*/
|
| +SQLITE_API int SQLITE_STDCALL sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){
|
| + int rc;
|
| + if( p ){
|
| +
|
| + /* Commit the transaction to the *-oal file. */
|
| + if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
|
| + p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg);
|
| + }
|
| +
|
| + rbuSaveState(p, p->eStage);
|
| +
|
| + if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
|
| + p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
|
| + }
|
| +
|
| + /* Close any open statement handles. */
|
| + rbuObjIterFinalize(&p->objiter);
|
| +
|
| + /* Close the open database handle and VFS object. */
|
| + sqlite3_close(p->dbMain);
|
| + sqlite3_close(p->dbRbu);
|
| + rbuDeleteVfs(p);
|
| + sqlite3_free(p->aBuf);
|
| + sqlite3_free(p->aFrame);
|
| +
|
| + rbuEditErrmsg(p);
|
| + rc = p->rc;
|
| + *pzErrmsg = p->zErrmsg;
|
| + sqlite3_free(p);
|
| + }else{
|
| + rc = SQLITE_NOMEM;
|
| + *pzErrmsg = 0;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Return the total number of key-value operations (inserts, deletes or
|
| +** updates) that have been performed on the target database since the
|
| +** current RBU update was started.
|
| +*/
|
| +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3rbu_progress(sqlite3rbu *pRbu){
|
| + return pRbu->nProgress;
|
| +}
|
| +
|
| +SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *p){
|
| + int rc = p->rc;
|
| +
|
| + if( rc==SQLITE_DONE ) return SQLITE_OK;
|
| +
|
| + assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE );
|
| + if( p->eStage==RBU_STAGE_OAL ){
|
| + assert( rc!=SQLITE_DONE );
|
| + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0);
|
| + }
|
| +
|
| + p->rc = rc;
|
| + rbuSaveState(p, p->eStage);
|
| + rc = p->rc;
|
| +
|
| + if( p->eStage==RBU_STAGE_OAL ){
|
| + assert( rc!=SQLITE_DONE );
|
| + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0);
|
| + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, 0);
|
| + if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "BEGIN IMMEDIATE", 0, 0,0);
|
| + }
|
| +
|
| + p->rc = rc;
|
| + return rc;
|
| +}
|
| +
|
| +/**************************************************************************
|
| +** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour
|
| +** of a standard VFS in the following ways:
|
| +**
|
| +** 1. Whenever the first page of a main database file is read or
|
| +** written, the value of the change-counter cookie is stored in
|
| +** rbu_file.iCookie. Similarly, the value of the "write-version"
|
| +** database header field is stored in rbu_file.iWriteVer. This ensures
|
| +** that the values are always trustworthy within an open transaction.
|
| +**
|
| +** 2. Whenever an SQLITE_OPEN_WAL file is opened, the (rbu_file.pWalFd)
|
| +** member variable of the associated database file descriptor is set
|
| +** to point to the new file. A mutex protected linked list of all main
|
| +** db fds opened using a particular RBU VFS is maintained at
|
| +** rbu_vfs.pMain to facilitate this.
|
| +**
|
| +** 3. Using a new file-control "SQLITE_FCNTL_RBU", a main db rbu_file
|
| +** object can be marked as the target database of an RBU update. This
|
| +** turns on the following extra special behaviour:
|
| +**
|
| +** 3a. If xAccess() is called to check if there exists a *-wal file
|
| +** associated with an RBU target database currently in RBU_STAGE_OAL
|
| +** stage (preparing the *-oal file), the following special handling
|
| +** applies:
|
| +**
|
| +** * if the *-wal file does exist, return SQLITE_CANTOPEN. An RBU
|
| +** target database may not be in wal mode already.
|
| +**
|
| +** * if the *-wal file does not exist, set the output parameter to
|
| +** non-zero (to tell SQLite that it does exist) anyway.
|
| +**
|
| +** Then, when xOpen() is called to open the *-wal file associated with
|
| +** the RBU target in RBU_STAGE_OAL stage, instead of opening the *-wal
|
| +** file, the rbu vfs opens the corresponding *-oal file instead.
|
| +**
|
| +** 3b. The *-shm pages returned by xShmMap() for a target db file in
|
| +** RBU_STAGE_OAL mode are actually stored in heap memory. This is to
|
| +** avoid creating a *-shm file on disk. Additionally, xShmLock() calls
|
| +** are no-ops on target database files in RBU_STAGE_OAL mode. This is
|
| +** because assert() statements in some VFS implementations fail if
|
| +** xShmLock() is called before xShmMap().
|
| +**
|
| +** 3c. If an EXCLUSIVE lock is attempted on a target database file in any
|
| +** mode except RBU_STAGE_DONE (all work completed and checkpointed), it
|
| +** fails with an SQLITE_BUSY error. This is to stop RBU connections
|
| +** from automatically checkpointing a *-wal (or *-oal) file from within
|
| +** sqlite3_close().
|
| +**
|
| +** 3d. In RBU_STAGE_CAPTURE mode, all xRead() calls on the wal file, and
|
| +** all xWrite() calls on the target database file perform no IO.
|
| +** Instead the frame and page numbers that would be read and written
|
| +** are recorded. Additionally, successful attempts to obtain exclusive
|
| +** xShmLock() WRITER, CHECKPOINTER and READ0 locks on the target
|
| +** database file are recorded. xShmLock() calls to unlock the same
|
| +** locks are no-ops (so that once obtained, these locks are never
|
| +** relinquished). Finally, calls to xSync() on the target database
|
| +** file fail with SQLITE_INTERNAL errors.
|
| +*/
|
| +
|
| +static void rbuUnlockShm(rbu_file *p){
|
| + if( p->pRbu ){
|
| + int (*xShmLock)(sqlite3_file*,int,int,int) = p->pReal->pMethods->xShmLock;
|
| + int i;
|
| + for(i=0; i<SQLITE_SHM_NLOCK;i++){
|
| + if( (1<<i) & p->pRbu->mLock ){
|
| + xShmLock(p->pReal, i, 1, SQLITE_SHM_UNLOCK|SQLITE_SHM_EXCLUSIVE);
|
| + }
|
| + }
|
| + p->pRbu->mLock = 0;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Close an rbu file.
|
| +*/
|
| +static int rbuVfsClose(sqlite3_file *pFile){
|
| + rbu_file *p = (rbu_file*)pFile;
|
| + int rc;
|
| + int i;
|
| +
|
| + /* Free the contents of the apShm[] array. And the array itself. */
|
| + for(i=0; i<p->nShm; i++){
|
| + sqlite3_free(p->apShm[i]);
|
| + }
|
| + sqlite3_free(p->apShm);
|
| + p->apShm = 0;
|
| + sqlite3_free(p->zDel);
|
| +
|
| + if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
|
| + rbu_file **pp;
|
| + sqlite3_mutex_enter(p->pRbuVfs->mutex);
|
| + for(pp=&p->pRbuVfs->pMain; *pp!=p; pp=&((*pp)->pMainNext));
|
| + *pp = p->pMainNext;
|
| + sqlite3_mutex_leave(p->pRbuVfs->mutex);
|
| + rbuUnlockShm(p);
|
| + p->pReal->pMethods->xShmUnmap(p->pReal, 0);
|
| + }
|
| +
|
| + /* Close the underlying file handle */
|
| + rc = p->pReal->pMethods->xClose(p->pReal);
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Read and return an unsigned 32-bit big-endian integer from the buffer
|
| +** passed as the only argument.
|
| +*/
|
| +static u32 rbuGetU32(u8 *aBuf){
|
| + return ((u32)aBuf[0] << 24)
|
| + + ((u32)aBuf[1] << 16)
|
| + + ((u32)aBuf[2] << 8)
|
| + + ((u32)aBuf[3]);
|
| +}
|
| +
|
| +/*
|
| +** Read data from an rbuVfs-file.
|
| +*/
|
| +static int rbuVfsRead(
|
| + sqlite3_file *pFile,
|
| + void *zBuf,
|
| + int iAmt,
|
| + sqlite_int64 iOfst
|
| +){
|
| + rbu_file *p = (rbu_file*)pFile;
|
| + sqlite3rbu *pRbu = p->pRbu;
|
| + int rc;
|
| +
|
| + if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){
|
| + assert( p->openFlags & SQLITE_OPEN_WAL );
|
| + rc = rbuCaptureWalRead(p->pRbu, iOfst, iAmt);
|
| + }else{
|
| + if( pRbu && pRbu->eStage==RBU_STAGE_OAL
|
| + && (p->openFlags & SQLITE_OPEN_WAL)
|
| + && iOfst>=pRbu->iOalSz
|
| + ){
|
| + rc = SQLITE_OK;
|
| + memset(zBuf, 0, iAmt);
|
| + }else{
|
| + rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
|
| + }
|
| + if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
|
| + /* These look like magic numbers. But they are stable, as they are part
|
| + ** of the definition of the SQLite file format, which may not change. */
|
| + u8 *pBuf = (u8*)zBuf;
|
| + p->iCookie = rbuGetU32(&pBuf[24]);
|
| + p->iWriteVer = pBuf[19];
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Write data to an rbuVfs-file.
|
| +*/
|
| +static int rbuVfsWrite(
|
| + sqlite3_file *pFile,
|
| + const void *zBuf,
|
| + int iAmt,
|
| + sqlite_int64 iOfst
|
| +){
|
| + rbu_file *p = (rbu_file*)pFile;
|
| + sqlite3rbu *pRbu = p->pRbu;
|
| + int rc;
|
| +
|
| + if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){
|
| + assert( p->openFlags & SQLITE_OPEN_MAIN_DB );
|
| + rc = rbuCaptureDbWrite(p->pRbu, iOfst);
|
| + }else{
|
| + if( pRbu && pRbu->eStage==RBU_STAGE_OAL
|
| + && (p->openFlags & SQLITE_OPEN_WAL)
|
| + && iOfst>=pRbu->iOalSz
|
| + ){
|
| + pRbu->iOalSz = iAmt + iOfst;
|
| + }
|
| + rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst);
|
| + if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
|
| + /* These look like magic numbers. But they are stable, as they are part
|
| + ** of the definition of the SQLite file format, which may not change. */
|
| + u8 *pBuf = (u8*)zBuf;
|
| + p->iCookie = rbuGetU32(&pBuf[24]);
|
| + p->iWriteVer = pBuf[19];
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Truncate an rbuVfs-file.
|
| +*/
|
| +static int rbuVfsTruncate(sqlite3_file *pFile, sqlite_int64 size){
|
| + rbu_file *p = (rbu_file*)pFile;
|
| + return p->pReal->pMethods->xTruncate(p->pReal, size);
|
| +}
|
| +
|
| +/*
|
| +** Sync an rbuVfs-file.
|
| +*/
|
| +static int rbuVfsSync(sqlite3_file *pFile, int flags){
|
| + rbu_file *p = (rbu_file *)pFile;
|
| + if( p->pRbu && p->pRbu->eStage==RBU_STAGE_CAPTURE ){
|
| + if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
|
| + return SQLITE_INTERNAL;
|
| + }
|
| + return SQLITE_OK;
|
| + }
|
| + return p->pReal->pMethods->xSync(p->pReal, flags);
|
| +}
|
| +
|
| +/*
|
| +** Return the current file-size of an rbuVfs-file.
|
| +*/
|
| +static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
|
| + rbu_file *p = (rbu_file *)pFile;
|
| + return p->pReal->pMethods->xFileSize(p->pReal, pSize);
|
| +}
|
| +
|
| +/*
|
| +** Lock an rbuVfs-file.
|
| +*/
|
| +static int rbuVfsLock(sqlite3_file *pFile, int eLock){
|
| + rbu_file *p = (rbu_file*)pFile;
|
| + sqlite3rbu *pRbu = p->pRbu;
|
| + int rc = SQLITE_OK;
|
| +
|
| + assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
|
| + if( pRbu && eLock==SQLITE_LOCK_EXCLUSIVE && pRbu->eStage!=RBU_STAGE_DONE ){
|
| + /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this
|
| + ** prevents it from checkpointing the database from sqlite3_close(). */
|
| + rc = SQLITE_BUSY;
|
| + }else{
|
| + rc = p->pReal->pMethods->xLock(p->pReal, eLock);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Unlock an rbuVfs-file.
|
| +*/
|
| +static int rbuVfsUnlock(sqlite3_file *pFile, int eLock){
|
| + rbu_file *p = (rbu_file *)pFile;
|
| + return p->pReal->pMethods->xUnlock(p->pReal, eLock);
|
| +}
|
| +
|
| +/*
|
| +** Check if another file-handle holds a RESERVED lock on an rbuVfs-file.
|
| +*/
|
| +static int rbuVfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){
|
| + rbu_file *p = (rbu_file *)pFile;
|
| + return p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut);
|
| +}
|
| +
|
| +/*
|
| +** File control method. For custom operations on an rbuVfs-file.
|
| +*/
|
| +static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){
|
| + rbu_file *p = (rbu_file *)pFile;
|
| + int (*xControl)(sqlite3_file*,int,void*) = p->pReal->pMethods->xFileControl;
|
| + int rc;
|
| +
|
| + assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB)
|
| + || p->openFlags & (SQLITE_OPEN_TRANSIENT_DB|SQLITE_OPEN_TEMP_JOURNAL)
|
| + );
|
| + if( op==SQLITE_FCNTL_RBU ){
|
| + sqlite3rbu *pRbu = (sqlite3rbu*)pArg;
|
| +
|
| + /* First try to find another RBU vfs lower down in the vfs stack. If
|
| + ** one is found, this vfs will operate in pass-through mode. The lower
|
| + ** level vfs will do the special RBU handling. */
|
| + rc = xControl(p->pReal, op, pArg);
|
| +
|
| + if( rc==SQLITE_NOTFOUND ){
|
| + /* Now search for a zipvfs instance lower down in the VFS stack. If
|
| + ** one is found, this is an error. */
|
| + void *dummy = 0;
|
| + rc = xControl(p->pReal, SQLITE_FCNTL_ZIPVFS, &dummy);
|
| + if( rc==SQLITE_OK ){
|
| + rc = SQLITE_ERROR;
|
| + pRbu->zErrmsg = sqlite3_mprintf("rbu/zipvfs setup error");
|
| + }else if( rc==SQLITE_NOTFOUND ){
|
| + pRbu->pTargetFd = p;
|
| + p->pRbu = pRbu;
|
| + if( p->pWalFd ) p->pWalFd->pRbu = pRbu;
|
| + rc = SQLITE_OK;
|
| + }
|
| + }
|
| + return rc;
|
| + }
|
| +
|
| + rc = xControl(p->pReal, op, pArg);
|
| + if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
|
| + rbu_vfs *pRbuVfs = p->pRbuVfs;
|
| + char *zIn = *(char**)pArg;
|
| + char *zOut = sqlite3_mprintf("rbu(%s)/%z", pRbuVfs->base.zName, zIn);
|
| + *(char**)pArg = zOut;
|
| + if( zOut==0 ) rc = SQLITE_NOMEM;
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Return the sector-size in bytes for an rbuVfs-file.
|
| +*/
|
| +static int rbuVfsSectorSize(sqlite3_file *pFile){
|
| + rbu_file *p = (rbu_file *)pFile;
|
| + return p->pReal->pMethods->xSectorSize(p->pReal);
|
| +}
|
| +
|
| +/*
|
| +** Return the device characteristic flags supported by an rbuVfs-file.
|
| +*/
|
| +static int rbuVfsDeviceCharacteristics(sqlite3_file *pFile){
|
| + rbu_file *p = (rbu_file *)pFile;
|
| + return p->pReal->pMethods->xDeviceCharacteristics(p->pReal);
|
| +}
|
| +
|
| +/*
|
| +** Take or release a shared-memory lock.
|
| +*/
|
| +static int rbuVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
|
| + rbu_file *p = (rbu_file*)pFile;
|
| + sqlite3rbu *pRbu = p->pRbu;
|
| + int rc = SQLITE_OK;
|
| +
|
| +#ifdef SQLITE_AMALGAMATION
|
| + assert( WAL_CKPT_LOCK==1 );
|
| +#endif
|
| +
|
| + assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
|
| + if( pRbu && (pRbu->eStage==RBU_STAGE_OAL || pRbu->eStage==RBU_STAGE_MOVE) ){
|
| + /* Magic number 1 is the WAL_CKPT_LOCK lock. Preventing SQLite from
|
| + ** taking this lock also prevents any checkpoints from occurring.
|
| + ** todo: really, it's not clear why this might occur, as
|
| + ** wal_autocheckpoint ought to be turned off. */
|
| + if( ofst==WAL_LOCK_CKPT && n==1 ) rc = SQLITE_BUSY;
|
| + }else{
|
| + int bCapture = 0;
|
| + if( n==1 && (flags & SQLITE_SHM_EXCLUSIVE)
|
| + && pRbu && pRbu->eStage==RBU_STAGE_CAPTURE
|
| + && (ofst==WAL_LOCK_WRITE || ofst==WAL_LOCK_CKPT || ofst==WAL_LOCK_READ0)
|
| + ){
|
| + bCapture = 1;
|
| + }
|
| +
|
| + if( bCapture==0 || 0==(flags & SQLITE_SHM_UNLOCK) ){
|
| + rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
|
| + if( bCapture && rc==SQLITE_OK ){
|
| + pRbu->mLock |= (1 << ofst);
|
| + }
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Obtain a pointer to a mapping of a single 32KiB page of the *-shm file.
|
| +*/
|
| +static int rbuVfsShmMap(
|
| + sqlite3_file *pFile,
|
| + int iRegion,
|
| + int szRegion,
|
| + int isWrite,
|
| + void volatile **pp
|
| +){
|
| + rbu_file *p = (rbu_file*)pFile;
|
| + int rc = SQLITE_OK;
|
| + int eStage = (p->pRbu ? p->pRbu->eStage : 0);
|
| +
|
| + /* If not in RBU_STAGE_OAL, allow this call to pass through. Or, if this
|
| + ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space
|
| + ** instead of a file on disk. */
|
| + assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
|
| + if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){
|
| + if( iRegion<=p->nShm ){
|
| + int nByte = (iRegion+1) * sizeof(char*);
|
| + char **apNew = (char**)sqlite3_realloc(p->apShm, nByte);
|
| + if( apNew==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm));
|
| + p->apShm = apNew;
|
| + p->nShm = iRegion+1;
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){
|
| + char *pNew = (char*)sqlite3_malloc(szRegion);
|
| + if( pNew==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + memset(pNew, 0, szRegion);
|
| + p->apShm[iRegion] = pNew;
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + *pp = p->apShm[iRegion];
|
| + }else{
|
| + *pp = 0;
|
| + }
|
| + }else{
|
| + assert( p->apShm==0 );
|
| + rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Memory barrier.
|
| +*/
|
| +static void rbuVfsShmBarrier(sqlite3_file *pFile){
|
| + rbu_file *p = (rbu_file *)pFile;
|
| + p->pReal->pMethods->xShmBarrier(p->pReal);
|
| +}
|
| +
|
| +/*
|
| +** The xShmUnmap method.
|
| +*/
|
| +static int rbuVfsShmUnmap(sqlite3_file *pFile, int delFlag){
|
| + rbu_file *p = (rbu_file*)pFile;
|
| + int rc = SQLITE_OK;
|
| + int eStage = (p->pRbu ? p->pRbu->eStage : 0);
|
| +
|
| + assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
|
| + if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){
|
| + /* no-op */
|
| + }else{
|
| + /* Release the checkpointer and writer locks */
|
| + rbuUnlockShm(p);
|
| + rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Given that zWal points to a buffer containing a wal file name passed to
|
| +** either the xOpen() or xAccess() VFS method, return a pointer to the
|
| +** file-handle opened by the same database connection on the corresponding
|
| +** database file.
|
| +*/
|
| +static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){
|
| + rbu_file *pDb;
|
| + sqlite3_mutex_enter(pRbuVfs->mutex);
|
| + for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext);
|
| + sqlite3_mutex_leave(pRbuVfs->mutex);
|
| + return pDb;
|
| +}
|
| +
|
| +/*
|
| +** Open an rbu file handle.
|
| +*/
|
| +static int rbuVfsOpen(
|
| + sqlite3_vfs *pVfs,
|
| + const char *zName,
|
| + sqlite3_file *pFile,
|
| + int flags,
|
| + int *pOutFlags
|
| +){
|
| + static sqlite3_io_methods rbuvfs_io_methods = {
|
| + 2, /* iVersion */
|
| + rbuVfsClose, /* xClose */
|
| + rbuVfsRead, /* xRead */
|
| + rbuVfsWrite, /* xWrite */
|
| + rbuVfsTruncate, /* xTruncate */
|
| + rbuVfsSync, /* xSync */
|
| + rbuVfsFileSize, /* xFileSize */
|
| + rbuVfsLock, /* xLock */
|
| + rbuVfsUnlock, /* xUnlock */
|
| + rbuVfsCheckReservedLock, /* xCheckReservedLock */
|
| + rbuVfsFileControl, /* xFileControl */
|
| + rbuVfsSectorSize, /* xSectorSize */
|
| + rbuVfsDeviceCharacteristics, /* xDeviceCharacteristics */
|
| + rbuVfsShmMap, /* xShmMap */
|
| + rbuVfsShmLock, /* xShmLock */
|
| + rbuVfsShmBarrier, /* xShmBarrier */
|
| + rbuVfsShmUnmap, /* xShmUnmap */
|
| + 0, 0 /* xFetch, xUnfetch */
|
| + };
|
| + rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs;
|
| + sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs;
|
| + rbu_file *pFd = (rbu_file *)pFile;
|
| + int rc = SQLITE_OK;
|
| + const char *zOpen = zName;
|
| +
|
| + memset(pFd, 0, sizeof(rbu_file));
|
| + pFd->pReal = (sqlite3_file*)&pFd[1];
|
| + pFd->pRbuVfs = pRbuVfs;
|
| + pFd->openFlags = flags;
|
| + if( zName ){
|
| + if( flags & SQLITE_OPEN_MAIN_DB ){
|
| + /* A main database has just been opened. The following block sets
|
| + ** (pFd->zWal) to point to a buffer owned by SQLite that contains
|
| + ** the name of the *-wal file this db connection will use. SQLite
|
| + ** happens to pass a pointer to this buffer when using xAccess()
|
| + ** or xOpen() to operate on the *-wal file. */
|
| + int n = strlen(zName);
|
| + const char *z = &zName[n];
|
| + if( flags & SQLITE_OPEN_URI ){
|
| + int odd = 0;
|
| + while( 1 ){
|
| + if( z[0]==0 ){
|
| + odd = 1 - odd;
|
| + if( odd && z[1]==0 ) break;
|
| + }
|
| + z++;
|
| + }
|
| + z += 2;
|
| + }else{
|
| + while( *z==0 ) z++;
|
| + }
|
| + z += (n + 8 + 1);
|
| + pFd->zWal = z;
|
| + }
|
| + else if( flags & SQLITE_OPEN_WAL ){
|
| + rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName);
|
| + if( pDb ){
|
| + if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
|
| + /* This call is to open a *-wal file. Intead, open the *-oal. This
|
| + ** code ensures that the string passed to xOpen() is terminated by a
|
| + ** pair of '\0' bytes in case the VFS attempts to extract a URI
|
| + ** parameter from it. */
|
| + int nCopy = strlen(zName);
|
| + char *zCopy = sqlite3_malloc(nCopy+2);
|
| + if( zCopy ){
|
| + memcpy(zCopy, zName, nCopy);
|
| + zCopy[nCopy-3] = 'o';
|
| + zCopy[nCopy] = '\0';
|
| + zCopy[nCopy+1] = '\0';
|
| + zOpen = (const char*)(pFd->zDel = zCopy);
|
| + }else{
|
| + rc = SQLITE_NOMEM;
|
| + }
|
| + pFd->pRbu = pDb->pRbu;
|
| + }
|
| + pDb->pWalFd = pFd;
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, flags, pOutFlags);
|
| + }
|
| + if( pFd->pReal->pMethods ){
|
| + /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods
|
| + ** pointer and, if the file is a main database file, link it into the
|
| + ** mutex protected linked list of all such files. */
|
| + pFile->pMethods = &rbuvfs_io_methods;
|
| + if( flags & SQLITE_OPEN_MAIN_DB ){
|
| + sqlite3_mutex_enter(pRbuVfs->mutex);
|
| + pFd->pMainNext = pRbuVfs->pMain;
|
| + pRbuVfs->pMain = pFd;
|
| + sqlite3_mutex_leave(pRbuVfs->mutex);
|
| + }
|
| + }else{
|
| + sqlite3_free(pFd->zDel);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Delete the file located at zPath.
|
| +*/
|
| +static int rbuVfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
|
| + sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
|
| + return pRealVfs->xDelete(pRealVfs, zPath, dirSync);
|
| +}
|
| +
|
| +/*
|
| +** Test for access permissions. Return true if the requested permission
|
| +** is available, or false otherwise.
|
| +*/
|
| +static int rbuVfsAccess(
|
| + sqlite3_vfs *pVfs,
|
| + const char *zPath,
|
| + int flags,
|
| + int *pResOut
|
| +){
|
| + rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs;
|
| + sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs;
|
| + int rc;
|
| +
|
| + rc = pRealVfs->xAccess(pRealVfs, zPath, flags, pResOut);
|
| +
|
| + /* If this call is to check if a *-wal file associated with an RBU target
|
| + ** database connection exists, and the RBU update is in RBU_STAGE_OAL,
|
| + ** the following special handling is activated:
|
| + **
|
| + ** a) if the *-wal file does exist, return SQLITE_CANTOPEN. This
|
| + ** ensures that the RBU extension never tries to update a database
|
| + ** in wal mode, even if the first page of the database file has
|
| + ** been damaged.
|
| + **
|
| + ** b) if the *-wal file does not exist, claim that it does anyway,
|
| + ** causing SQLite to call xOpen() to open it. This call will also
|
| + ** be intercepted (see the rbuVfsOpen() function) and the *-oal
|
| + ** file opened instead.
|
| + */
|
| + if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS ){
|
| + rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath);
|
| + if( pDb && pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
|
| + if( *pResOut ){
|
| + rc = SQLITE_CANTOPEN;
|
| + }else{
|
| + *pResOut = 1;
|
| + }
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Populate buffer zOut with the full canonical pathname corresponding
|
| +** to the pathname in zPath. zOut is guaranteed to point to a buffer
|
| +** of at least (DEVSYM_MAX_PATHNAME+1) bytes.
|
| +*/
|
| +static int rbuVfsFullPathname(
|
| + sqlite3_vfs *pVfs,
|
| + const char *zPath,
|
| + int nOut,
|
| + char *zOut
|
| +){
|
| + sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
|
| + return pRealVfs->xFullPathname(pRealVfs, zPath, nOut, zOut);
|
| +}
|
| +
|
| +#ifndef SQLITE_OMIT_LOAD_EXTENSION
|
| +/*
|
| +** Open the dynamic library located at zPath and return a handle.
|
| +*/
|
| +static void *rbuVfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){
|
| + sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
|
| + return pRealVfs->xDlOpen(pRealVfs, zPath);
|
| +}
|
| +
|
| +/*
|
| +** Populate the buffer zErrMsg (size nByte bytes) with a human readable
|
| +** utf-8 string describing the most recent error encountered associated
|
| +** with dynamic libraries.
|
| +*/
|
| +static void rbuVfsDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
|
| + sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
|
| + pRealVfs->xDlError(pRealVfs, nByte, zErrMsg);
|
| +}
|
| +
|
| +/*
|
| +** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
|
| +*/
|
| +static void (*rbuVfsDlSym(
|
| + sqlite3_vfs *pVfs,
|
| + void *pArg,
|
| + const char *zSym
|
| +))(void){
|
| + sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
|
| + return pRealVfs->xDlSym(pRealVfs, pArg, zSym);
|
| +}
|
| +
|
| +/*
|
| +** Close the dynamic library handle pHandle.
|
| +*/
|
| +static void rbuVfsDlClose(sqlite3_vfs *pVfs, void *pHandle){
|
| + sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
|
| + pRealVfs->xDlClose(pRealVfs, pHandle);
|
| +}
|
| +#endif /* SQLITE_OMIT_LOAD_EXTENSION */
|
| +
|
| +/*
|
| +** Populate the buffer pointed to by zBufOut with nByte bytes of
|
| +** random data.
|
| +*/
|
| +static int rbuVfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
|
| + sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
|
| + return pRealVfs->xRandomness(pRealVfs, nByte, zBufOut);
|
| +}
|
| +
|
| +/*
|
| +** Sleep for nMicro microseconds. Return the number of microseconds
|
| +** actually slept.
|
| +*/
|
| +static int rbuVfsSleep(sqlite3_vfs *pVfs, int nMicro){
|
| + sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
|
| + return pRealVfs->xSleep(pRealVfs, nMicro);
|
| +}
|
| +
|
| +/*
|
| +** Return the current time as a Julian Day number in *pTimeOut.
|
| +*/
|
| +static int rbuVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
|
| + sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
|
| + return pRealVfs->xCurrentTime(pRealVfs, pTimeOut);
|
| +}
|
| +
|
| +/*
|
| +** No-op.
|
| +*/
|
| +static int rbuVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){
|
| + return 0;
|
| +}
|
| +
|
| +/*
|
| +** Deregister and destroy an RBU vfs created by an earlier call to
|
| +** sqlite3rbu_create_vfs().
|
| +*/
|
| +SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName){
|
| + sqlite3_vfs *pVfs = sqlite3_vfs_find(zName);
|
| + if( pVfs && pVfs->xOpen==rbuVfsOpen ){
|
| + sqlite3_mutex_free(((rbu_vfs*)pVfs)->mutex);
|
| + sqlite3_vfs_unregister(pVfs);
|
| + sqlite3_free(pVfs);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Create an RBU VFS named zName that accesses the underlying file-system
|
| +** via existing VFS zParent. The new object is registered as a non-default
|
| +** VFS with SQLite before returning.
|
| +*/
|
| +SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const char *zParent){
|
| +
|
| + /* Template for VFS */
|
| + static sqlite3_vfs vfs_template = {
|
| + 1, /* iVersion */
|
| + 0, /* szOsFile */
|
| + 0, /* mxPathname */
|
| + 0, /* pNext */
|
| + 0, /* zName */
|
| + 0, /* pAppData */
|
| + rbuVfsOpen, /* xOpen */
|
| + rbuVfsDelete, /* xDelete */
|
| + rbuVfsAccess, /* xAccess */
|
| + rbuVfsFullPathname, /* xFullPathname */
|
| +
|
| +#ifndef SQLITE_OMIT_LOAD_EXTENSION
|
| + rbuVfsDlOpen, /* xDlOpen */
|
| + rbuVfsDlError, /* xDlError */
|
| + rbuVfsDlSym, /* xDlSym */
|
| + rbuVfsDlClose, /* xDlClose */
|
| +#else
|
| + 0, 0, 0, 0,
|
| +#endif
|
| +
|
| + rbuVfsRandomness, /* xRandomness */
|
| + rbuVfsSleep, /* xSleep */
|
| + rbuVfsCurrentTime, /* xCurrentTime */
|
| + rbuVfsGetLastError, /* xGetLastError */
|
| + 0, /* xCurrentTimeInt64 (version 2) */
|
| + 0, 0, 0 /* Unimplemented version 3 methods */
|
| + };
|
| +
|
| + rbu_vfs *pNew = 0; /* Newly allocated VFS */
|
| + int nName;
|
| + int rc = SQLITE_OK;
|
| +
|
| + int nByte;
|
| + nName = strlen(zName);
|
| + nByte = sizeof(rbu_vfs) + nName + 1;
|
| + pNew = (rbu_vfs*)sqlite3_malloc(nByte);
|
| + if( pNew==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + sqlite3_vfs *pParent; /* Parent VFS */
|
| + memset(pNew, 0, nByte);
|
| + pParent = sqlite3_vfs_find(zParent);
|
| + if( pParent==0 ){
|
| + rc = SQLITE_NOTFOUND;
|
| + }else{
|
| + char *zSpace;
|
| + memcpy(&pNew->base, &vfs_template, sizeof(sqlite3_vfs));
|
| + pNew->base.mxPathname = pParent->mxPathname;
|
| + pNew->base.szOsFile = sizeof(rbu_file) + pParent->szOsFile;
|
| + pNew->pRealVfs = pParent;
|
| + pNew->base.zName = (const char*)(zSpace = (char*)&pNew[1]);
|
| + memcpy(zSpace, zName, nName);
|
| +
|
| + /* Allocate the mutex and register the new VFS (not as the default) */
|
| + pNew->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE);
|
| + if( pNew->mutex==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + rc = sqlite3_vfs_register(&pNew->base, 0);
|
| + }
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + sqlite3_mutex_free(pNew->mutex);
|
| + sqlite3_free(pNew);
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/**************************************************************************/
|
| +
|
| +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) */
|
| +
|
| +/************** End of sqlite3rbu.c ******************************************/
|
| +/************** Begin file dbstat.c ******************************************/
|
| +/*
|
| +** 2010 July 12
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +** This file contains an implementation of the "dbstat" virtual table.
|
| +**
|
| +** The dbstat virtual table is used to extract low-level formatting
|
| +** information from an SQLite database in order to implement the
|
| +** "sqlite3_analyzer" utility. See the ../tool/spaceanal.tcl script
|
| +** for an example implementation.
|
| +**
|
| +** Additional information is available on the "dbstat.html" page of the
|
| +** official SQLite documentation.
|
| +*/
|
| +
|
| +/* #include "sqliteInt.h" ** Requires access to internal data structures ** */
|
| +#if (defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST)) \
|
| + && !defined(SQLITE_OMIT_VIRTUALTABLE)
|
| +
|
| +/*
|
| +** Page paths:
|
| +**
|
| +** The value of the 'path' column describes the path taken from the
|
| +** root-node of the b-tree structure to each page. The value of the
|
| +** root-node path is '/'.
|
| +**
|
| +** The value of the path for the left-most child page of the root of
|
| +** a b-tree is '/000/'. (Btrees store content ordered from left to right
|
| +** so the pages to the left have smaller keys than the pages to the right.)
|
| +** The next to left-most child of the root page is
|
| +** '/001', and so on, each sibling page identified by a 3-digit hex
|
| +** value. The children of the 451st left-most sibling have paths such
|
| +** as '/1c2/000/, '/1c2/001/' etc.
|
| +**
|
| +** Overflow pages are specified by appending a '+' character and a
|
| +** six-digit hexadecimal value to the path to the cell they are linked
|
| +** from. For example, the three overflow pages in a chain linked from
|
| +** the left-most cell of the 450th child of the root page are identified
|
| +** by the paths:
|
| +**
|
| +** '/1c2/000+000000' // First page in overflow chain
|
| +** '/1c2/000+000001' // Second page in overflow chain
|
| +** '/1c2/000+000002' // Third page in overflow chain
|
| +**
|
| +** If the paths are sorted using the BINARY collation sequence, then
|
| +** the overflow pages associated with a cell will appear earlier in the
|
| +** sort-order than its child page:
|
| +**
|
| +** '/1c2/000/' // Left-most child of 451st child of root
|
| +*/
|
| +#define VTAB_SCHEMA \
|
| + "CREATE TABLE xx( " \
|
| + " name STRING, /* Name of table or index */" \
|
| + " path INTEGER, /* Path to page from root */" \
|
| + " pageno INTEGER, /* Page number */" \
|
| + " pagetype STRING, /* 'internal', 'leaf' or 'overflow' */" \
|
| + " ncell INTEGER, /* Cells on page (0 for overflow) */" \
|
| + " payload INTEGER, /* Bytes of payload on this page */" \
|
| + " unused INTEGER, /* Bytes of unused space on this page */" \
|
| + " mx_payload INTEGER, /* Largest payload size of all cells */" \
|
| + " pgoffset INTEGER, /* Offset of page in file */" \
|
| + " pgsize INTEGER, /* Size of the page */" \
|
| + " schema TEXT HIDDEN /* Database schema being analyzed */" \
|
| + ");"
|
| +
|
| +
|
| +typedef struct StatTable StatTable;
|
| +typedef struct StatCursor StatCursor;
|
| +typedef struct StatPage StatPage;
|
| +typedef struct StatCell StatCell;
|
| +
|
| +struct StatCell {
|
| + int nLocal; /* Bytes of local payload */
|
| + u32 iChildPg; /* Child node (or 0 if this is a leaf) */
|
| + int nOvfl; /* Entries in aOvfl[] */
|
| + u32 *aOvfl; /* Array of overflow page numbers */
|
| + int nLastOvfl; /* Bytes of payload on final overflow page */
|
| + int iOvfl; /* Iterates through aOvfl[] */
|
| +};
|
| +
|
| +struct StatPage {
|
| + u32 iPgno;
|
| + DbPage *pPg;
|
| + int iCell;
|
| +
|
| + char *zPath; /* Path to this page */
|
| +
|
| + /* Variables populated by statDecodePage(): */
|
| + u8 flags; /* Copy of flags byte */
|
| + int nCell; /* Number of cells on page */
|
| + int nUnused; /* Number of unused bytes on page */
|
| + StatCell *aCell; /* Array of parsed cells */
|
| + u32 iRightChildPg; /* Right-child page number (or 0) */
|
| + int nMxPayload; /* Largest payload of any cell on this page */
|
| +};
|
| +
|
| +struct StatCursor {
|
| + sqlite3_vtab_cursor base;
|
| + sqlite3_stmt *pStmt; /* Iterates through set of root pages */
|
| + int isEof; /* After pStmt has returned SQLITE_DONE */
|
| + int iDb; /* Schema used for this query */
|
| +
|
| + StatPage aPage[32];
|
| + int iPage; /* Current entry in aPage[] */
|
| +
|
| + /* Values to return. */
|
| + char *zName; /* Value of 'name' column */
|
| + char *zPath; /* Value of 'path' column */
|
| + u32 iPageno; /* Value of 'pageno' column */
|
| + char *zPagetype; /* Value of 'pagetype' column */
|
| + int nCell; /* Value of 'ncell' column */
|
| + int nPayload; /* Value of 'payload' column */
|
| + int nUnused; /* Value of 'unused' column */
|
| + int nMxPayload; /* Value of 'mx_payload' column */
|
| + i64 iOffset; /* Value of 'pgOffset' column */
|
| + int szPage; /* Value of 'pgSize' column */
|
| +};
|
| +
|
| +struct StatTable {
|
| + sqlite3_vtab base;
|
| + sqlite3 *db;
|
| + int iDb; /* Index of database to analyze */
|
| +};
|
| +
|
| +#ifndef get2byte
|
| +# define get2byte(x) ((x)[0]<<8 | (x)[1])
|
| +#endif
|
| +
|
| +/*
|
| +** Connect to or create a statvfs virtual table.
|
| +*/
|
| +static int statConnect(
|
| + sqlite3 *db,
|
| + void *pAux,
|
| + int argc, const char *const*argv,
|
| + sqlite3_vtab **ppVtab,
|
| + char **pzErr
|
| +){
|
| + StatTable *pTab = 0;
|
| + int rc = SQLITE_OK;
|
| + int iDb;
|
| +
|
| + if( argc>=4 ){
|
| + iDb = sqlite3FindDbName(db, argv[3]);
|
| + if( iDb<0 ){
|
| + *pzErr = sqlite3_mprintf("no such database: %s", argv[3]);
|
| + return SQLITE_ERROR;
|
| + }
|
| + }else{
|
| + iDb = 0;
|
| + }
|
| + rc = sqlite3_declare_vtab(db, VTAB_SCHEMA);
|
| + if( rc==SQLITE_OK ){
|
| + pTab = (StatTable *)sqlite3_malloc64(sizeof(StatTable));
|
| + if( pTab==0 ) rc = SQLITE_NOMEM;
|
| + }
|
| +
|
| + assert( rc==SQLITE_OK || pTab==0 );
|
| + if( rc==SQLITE_OK ){
|
| + memset(pTab, 0, sizeof(StatTable));
|
| + pTab->db = db;
|
| + pTab->iDb = iDb;
|
| + }
|
| +
|
| + *ppVtab = (sqlite3_vtab*)pTab;
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Disconnect from or destroy a statvfs virtual table.
|
| +*/
|
| +static int statDisconnect(sqlite3_vtab *pVtab){
|
| + sqlite3_free(pVtab);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** There is no "best-index". This virtual table always does a linear
|
| +** scan. However, a schema=? constraint should cause this table to
|
| +** operate on a different database schema, so check for it.
|
| +**
|
| +** idxNum is normally 0, but will be 1 if a schema=? constraint exists.
|
| +*/
|
| +static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
|
| + int i;
|
| +
|
| + pIdxInfo->estimatedCost = 1.0e6; /* Initial cost estimate */
|
| +
|
| + /* Look for a valid schema=? constraint. If found, change the idxNum to
|
| + ** 1 and request the value of that constraint be sent to xFilter. And
|
| + ** lower the cost estimate to encourage the constrained version to be
|
| + ** used.
|
| + */
|
| + for(i=0; i<pIdxInfo->nConstraint; i++){
|
| + if( pIdxInfo->aConstraint[i].usable==0 ) continue;
|
| + if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
|
| + if( pIdxInfo->aConstraint[i].iColumn!=10 ) continue;
|
| + pIdxInfo->idxNum = 1;
|
| + pIdxInfo->estimatedCost = 1.0;
|
| + pIdxInfo->aConstraintUsage[i].argvIndex = 1;
|
| + pIdxInfo->aConstraintUsage[i].omit = 1;
|
| + break;
|
| + }
|
| +
|
| +
|
| + /* Records are always returned in ascending order of (name, path).
|
| + ** If this will satisfy the client, set the orderByConsumed flag so that
|
| + ** SQLite does not do an external sort.
|
| + */
|
| + if( ( pIdxInfo->nOrderBy==1
|
| + && pIdxInfo->aOrderBy[0].iColumn==0
|
| + && pIdxInfo->aOrderBy[0].desc==0
|
| + ) ||
|
| + ( pIdxInfo->nOrderBy==2
|
| + && pIdxInfo->aOrderBy[0].iColumn==0
|
| + && pIdxInfo->aOrderBy[0].desc==0
|
| + && pIdxInfo->aOrderBy[1].iColumn==1
|
| + && pIdxInfo->aOrderBy[1].desc==0
|
| + )
|
| + ){
|
| + pIdxInfo->orderByConsumed = 1;
|
| + }
|
| +
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Open a new statvfs cursor.
|
| +*/
|
| +static int statOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
|
| + StatTable *pTab = (StatTable *)pVTab;
|
| + StatCursor *pCsr;
|
| +
|
| + pCsr = (StatCursor *)sqlite3_malloc64(sizeof(StatCursor));
|
| + if( pCsr==0 ){
|
| + return SQLITE_NOMEM;
|
| + }else{
|
| + memset(pCsr, 0, sizeof(StatCursor));
|
| + pCsr->base.pVtab = pVTab;
|
| + pCsr->iDb = pTab->iDb;
|
| + }
|
| +
|
| + *ppCursor = (sqlite3_vtab_cursor *)pCsr;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static void statClearPage(StatPage *p){
|
| + int i;
|
| + if( p->aCell ){
|
| + for(i=0; i<p->nCell; i++){
|
| + sqlite3_free(p->aCell[i].aOvfl);
|
| + }
|
| + sqlite3_free(p->aCell);
|
| + }
|
| + sqlite3PagerUnref(p->pPg);
|
| + sqlite3_free(p->zPath);
|
| + memset(p, 0, sizeof(StatPage));
|
| +}
|
| +
|
| +static void statResetCsr(StatCursor *pCsr){
|
| + int i;
|
| + sqlite3_reset(pCsr->pStmt);
|
| + for(i=0; i<ArraySize(pCsr->aPage); i++){
|
| + statClearPage(&pCsr->aPage[i]);
|
| + }
|
| + pCsr->iPage = 0;
|
| + sqlite3_free(pCsr->zPath);
|
| + pCsr->zPath = 0;
|
| + pCsr->isEof = 0;
|
| +}
|
| +
|
| +/*
|
| +** Close a statvfs cursor.
|
| +*/
|
| +static int statClose(sqlite3_vtab_cursor *pCursor){
|
| + StatCursor *pCsr = (StatCursor *)pCursor;
|
| + statResetCsr(pCsr);
|
| + sqlite3_finalize(pCsr->pStmt);
|
| + sqlite3_free(pCsr);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static void getLocalPayload(
|
| + int nUsable, /* Usable bytes per page */
|
| + u8 flags, /* Page flags */
|
| + int nTotal, /* Total record (payload) size */
|
| + int *pnLocal /* OUT: Bytes stored locally */
|
| +){
|
| + int nLocal;
|
| + int nMinLocal;
|
| + int nMaxLocal;
|
| +
|
| + if( flags==0x0D ){ /* Table leaf node */
|
| + nMinLocal = (nUsable - 12) * 32 / 255 - 23;
|
| + nMaxLocal = nUsable - 35;
|
| + }else{ /* Index interior and leaf nodes */
|
| + nMinLocal = (nUsable - 12) * 32 / 255 - 23;
|
| + nMaxLocal = (nUsable - 12) * 64 / 255 - 23;
|
| + }
|
| +
|
| + nLocal = nMinLocal + (nTotal - nMinLocal) % (nUsable - 4);
|
| + if( nLocal>nMaxLocal ) nLocal = nMinLocal;
|
| + *pnLocal = nLocal;
|
| +}
|
| +
|
| +static int statDecodePage(Btree *pBt, StatPage *p){
|
| + int nUnused;
|
| + int iOff;
|
| + int nHdr;
|
| + int isLeaf;
|
| + int szPage;
|
| +
|
| + u8 *aData = sqlite3PagerGetData(p->pPg);
|
| + u8 *aHdr = &aData[p->iPgno==1 ? 100 : 0];
|
| +
|
| + p->flags = aHdr[0];
|
| + p->nCell = get2byte(&aHdr[3]);
|
| + p->nMxPayload = 0;
|
| +
|
| + isLeaf = (p->flags==0x0A || p->flags==0x0D);
|
| + nHdr = 12 - isLeaf*4 + (p->iPgno==1)*100;
|
| +
|
| + nUnused = get2byte(&aHdr[5]) - nHdr - 2*p->nCell;
|
| + nUnused += (int)aHdr[7];
|
| + iOff = get2byte(&aHdr[1]);
|
| + while( iOff ){
|
| + nUnused += get2byte(&aData[iOff+2]);
|
| + iOff = get2byte(&aData[iOff]);
|
| + }
|
| + p->nUnused = nUnused;
|
| + p->iRightChildPg = isLeaf ? 0 : sqlite3Get4byte(&aHdr[8]);
|
| + szPage = sqlite3BtreeGetPageSize(pBt);
|
| +
|
| + if( p->nCell ){
|
| + int i; /* Used to iterate through cells */
|
| + int nUsable; /* Usable bytes per page */
|
| +
|
| + sqlite3BtreeEnter(pBt);
|
| + nUsable = szPage - sqlite3BtreeGetReserveNoMutex(pBt);
|
| + sqlite3BtreeLeave(pBt);
|
| + p->aCell = sqlite3_malloc64((p->nCell+1) * sizeof(StatCell));
|
| + if( p->aCell==0 ) return SQLITE_NOMEM;
|
| + memset(p->aCell, 0, (p->nCell+1) * sizeof(StatCell));
|
| +
|
| + for(i=0; i<p->nCell; i++){
|
| + StatCell *pCell = &p->aCell[i];
|
| +
|
| + iOff = get2byte(&aData[nHdr+i*2]);
|
| + if( !isLeaf ){
|
| + pCell->iChildPg = sqlite3Get4byte(&aData[iOff]);
|
| + iOff += 4;
|
| + }
|
| + if( p->flags==0x05 ){
|
| + /* A table interior node. nPayload==0. */
|
| + }else{
|
| + u32 nPayload; /* Bytes of payload total (local+overflow) */
|
| + int nLocal; /* Bytes of payload stored locally */
|
| + iOff += getVarint32(&aData[iOff], nPayload);
|
| + if( p->flags==0x0D ){
|
| + u64 dummy;
|
| + iOff += sqlite3GetVarint(&aData[iOff], &dummy);
|
| + }
|
| + if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload;
|
| + getLocalPayload(nUsable, p->flags, nPayload, &nLocal);
|
| + pCell->nLocal = nLocal;
|
| + assert( nLocal>=0 );
|
| + assert( nPayload>=(u32)nLocal );
|
| + assert( nLocal<=(nUsable-35) );
|
| + if( nPayload>(u32)nLocal ){
|
| + int j;
|
| + int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4);
|
| + pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4);
|
| + pCell->nOvfl = nOvfl;
|
| + pCell->aOvfl = sqlite3_malloc64(sizeof(u32)*nOvfl);
|
| + if( pCell->aOvfl==0 ) return SQLITE_NOMEM;
|
| + pCell->aOvfl[0] = sqlite3Get4byte(&aData[iOff+nLocal]);
|
| + for(j=1; j<nOvfl; j++){
|
| + int rc;
|
| + u32 iPrev = pCell->aOvfl[j-1];
|
| + DbPage *pPg = 0;
|
| + rc = sqlite3PagerGet(sqlite3BtreePager(pBt), iPrev, &pPg, 0);
|
| + if( rc!=SQLITE_OK ){
|
| + assert( pPg==0 );
|
| + return rc;
|
| + }
|
| + pCell->aOvfl[j] = sqlite3Get4byte(sqlite3PagerGetData(pPg));
|
| + sqlite3PagerUnref(pPg);
|
| + }
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Populate the pCsr->iOffset and pCsr->szPage member variables. Based on
|
| +** the current value of pCsr->iPageno.
|
| +*/
|
| +static void statSizeAndOffset(StatCursor *pCsr){
|
| + StatTable *pTab = (StatTable *)((sqlite3_vtab_cursor *)pCsr)->pVtab;
|
| + Btree *pBt = pTab->db->aDb[pTab->iDb].pBt;
|
| + Pager *pPager = sqlite3BtreePager(pBt);
|
| + sqlite3_file *fd;
|
| + sqlite3_int64 x[2];
|
| +
|
| + /* The default page size and offset */
|
| + pCsr->szPage = sqlite3BtreeGetPageSize(pBt);
|
| + pCsr->iOffset = (i64)pCsr->szPage * (pCsr->iPageno - 1);
|
| +
|
| + /* If connected to a ZIPVFS backend, override the page size and
|
| + ** offset with actual values obtained from ZIPVFS.
|
| + */
|
| + fd = sqlite3PagerFile(pPager);
|
| + x[0] = pCsr->iPageno;
|
| + if( fd->pMethods!=0 && sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){
|
| + pCsr->iOffset = x[0];
|
| + pCsr->szPage = (int)x[1];
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Move a statvfs cursor to the next entry in the file.
|
| +*/
|
| +static int statNext(sqlite3_vtab_cursor *pCursor){
|
| + int rc;
|
| + int nPayload;
|
| + char *z;
|
| + StatCursor *pCsr = (StatCursor *)pCursor;
|
| + StatTable *pTab = (StatTable *)pCursor->pVtab;
|
| + Btree *pBt = pTab->db->aDb[pCsr->iDb].pBt;
|
| + Pager *pPager = sqlite3BtreePager(pBt);
|
| +
|
| + sqlite3_free(pCsr->zPath);
|
| + pCsr->zPath = 0;
|
| +
|
| +statNextRestart:
|
| + if( pCsr->aPage[0].pPg==0 ){
|
| + rc = sqlite3_step(pCsr->pStmt);
|
| + if( rc==SQLITE_ROW ){
|
| + int nPage;
|
| + u32 iRoot = (u32)sqlite3_column_int64(pCsr->pStmt, 1);
|
| + sqlite3PagerPagecount(pPager, &nPage);
|
| + if( nPage==0 ){
|
| + pCsr->isEof = 1;
|
| + return sqlite3_reset(pCsr->pStmt);
|
| + }
|
| + rc = sqlite3PagerGet(pPager, iRoot, &pCsr->aPage[0].pPg, 0);
|
| + pCsr->aPage[0].iPgno = iRoot;
|
| + pCsr->aPage[0].iCell = 0;
|
| + pCsr->aPage[0].zPath = z = sqlite3_mprintf("/");
|
| + pCsr->iPage = 0;
|
| + if( z==0 ) rc = SQLITE_NOMEM;
|
| + }else{
|
| + pCsr->isEof = 1;
|
| + return sqlite3_reset(pCsr->pStmt);
|
| + }
|
| + }else{
|
| +
|
| + /* Page p itself has already been visited. */
|
| + StatPage *p = &pCsr->aPage[pCsr->iPage];
|
| +
|
| + while( p->iCell<p->nCell ){
|
| + StatCell *pCell = &p->aCell[p->iCell];
|
| + if( pCell->iOvfl<pCell->nOvfl ){
|
| + int nUsable;
|
| + sqlite3BtreeEnter(pBt);
|
| + nUsable = sqlite3BtreeGetPageSize(pBt) -
|
| + sqlite3BtreeGetReserveNoMutex(pBt);
|
| + sqlite3BtreeLeave(pBt);
|
| + pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0);
|
| + pCsr->iPageno = pCell->aOvfl[pCell->iOvfl];
|
| + pCsr->zPagetype = "overflow";
|
| + pCsr->nCell = 0;
|
| + pCsr->nMxPayload = 0;
|
| + pCsr->zPath = z = sqlite3_mprintf(
|
| + "%s%.3x+%.6x", p->zPath, p->iCell, pCell->iOvfl
|
| + );
|
| + if( pCell->iOvfl<pCell->nOvfl-1 ){
|
| + pCsr->nUnused = 0;
|
| + pCsr->nPayload = nUsable - 4;
|
| + }else{
|
| + pCsr->nPayload = pCell->nLastOvfl;
|
| + pCsr->nUnused = nUsable - 4 - pCsr->nPayload;
|
| + }
|
| + pCell->iOvfl++;
|
| + statSizeAndOffset(pCsr);
|
| + return z==0 ? SQLITE_NOMEM : SQLITE_OK;
|
| + }
|
| + if( p->iRightChildPg ) break;
|
| + p->iCell++;
|
| + }
|
| +
|
| + if( !p->iRightChildPg || p->iCell>p->nCell ){
|
| + statClearPage(p);
|
| + if( pCsr->iPage==0 ) return statNext(pCursor);
|
| + pCsr->iPage--;
|
| + goto statNextRestart; /* Tail recursion */
|
| + }
|
| + pCsr->iPage++;
|
| + assert( p==&pCsr->aPage[pCsr->iPage-1] );
|
| +
|
| + if( p->iCell==p->nCell ){
|
| + p[1].iPgno = p->iRightChildPg;
|
| + }else{
|
| + p[1].iPgno = p->aCell[p->iCell].iChildPg;
|
| + }
|
| + rc = sqlite3PagerGet(pPager, p[1].iPgno, &p[1].pPg, 0);
|
| + p[1].iCell = 0;
|
| + p[1].zPath = z = sqlite3_mprintf("%s%.3x/", p->zPath, p->iCell);
|
| + p->iCell++;
|
| + if( z==0 ) rc = SQLITE_NOMEM;
|
| + }
|
| +
|
| +
|
| + /* Populate the StatCursor fields with the values to be returned
|
| + ** by the xColumn() and xRowid() methods.
|
| + */
|
| + if( rc==SQLITE_OK ){
|
| + int i;
|
| + StatPage *p = &pCsr->aPage[pCsr->iPage];
|
| + pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0);
|
| + pCsr->iPageno = p->iPgno;
|
| +
|
| + rc = statDecodePage(pBt, p);
|
| + if( rc==SQLITE_OK ){
|
| + statSizeAndOffset(pCsr);
|
| +
|
| + switch( p->flags ){
|
| + case 0x05: /* table internal */
|
| + case 0x02: /* index internal */
|
| + pCsr->zPagetype = "internal";
|
| + break;
|
| + case 0x0D: /* table leaf */
|
| + case 0x0A: /* index leaf */
|
| + pCsr->zPagetype = "leaf";
|
| + break;
|
| + default:
|
| + pCsr->zPagetype = "corrupted";
|
| + break;
|
| + }
|
| + pCsr->nCell = p->nCell;
|
| + pCsr->nUnused = p->nUnused;
|
| + pCsr->nMxPayload = p->nMxPayload;
|
| + pCsr->zPath = z = sqlite3_mprintf("%s", p->zPath);
|
| + if( z==0 ) rc = SQLITE_NOMEM;
|
| + nPayload = 0;
|
| + for(i=0; i<p->nCell; i++){
|
| + nPayload += p->aCell[i].nLocal;
|
| + }
|
| + pCsr->nPayload = nPayload;
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +static int statEof(sqlite3_vtab_cursor *pCursor){
|
| + StatCursor *pCsr = (StatCursor *)pCursor;
|
| + return pCsr->isEof;
|
| +}
|
| +
|
| +static int statFilter(
|
| + sqlite3_vtab_cursor *pCursor,
|
| + int idxNum, const char *idxStr,
|
| + int argc, sqlite3_value **argv
|
| +){
|
| + StatCursor *pCsr = (StatCursor *)pCursor;
|
| + StatTable *pTab = (StatTable*)(pCursor->pVtab);
|
| + char *zSql;
|
| + int rc = SQLITE_OK;
|
| + char *zMaster;
|
| +
|
| + if( idxNum==1 ){
|
| + const char *zDbase = (const char*)sqlite3_value_text(argv[0]);
|
| + pCsr->iDb = sqlite3FindDbName(pTab->db, zDbase);
|
| + if( pCsr->iDb<0 ){
|
| + sqlite3_free(pCursor->pVtab->zErrMsg);
|
| + pCursor->pVtab->zErrMsg = sqlite3_mprintf("no such schema: %s", zDbase);
|
| + return pCursor->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM;
|
| + }
|
| + }else{
|
| + pCsr->iDb = pTab->iDb;
|
| + }
|
| + statResetCsr(pCsr);
|
| + sqlite3_finalize(pCsr->pStmt);
|
| + pCsr->pStmt = 0;
|
| + zMaster = pCsr->iDb==1 ? "sqlite_temp_master" : "sqlite_master";
|
| + zSql = sqlite3_mprintf(
|
| + "SELECT 'sqlite_master' AS name, 1 AS rootpage, 'table' AS type"
|
| + " UNION ALL "
|
| + "SELECT name, rootpage, type"
|
| + " FROM \"%w\".%s WHERE rootpage!=0"
|
| + " ORDER BY name", pTab->db->aDb[pCsr->iDb].zName, zMaster);
|
| + if( zSql==0 ){
|
| + return SQLITE_NOMEM;
|
| + }else{
|
| + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
|
| + sqlite3_free(zSql);
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = statNext(pCursor);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static int statColumn(
|
| + sqlite3_vtab_cursor *pCursor,
|
| + sqlite3_context *ctx,
|
| + int i
|
| +){
|
| + StatCursor *pCsr = (StatCursor *)pCursor;
|
| + switch( i ){
|
| + case 0: /* name */
|
| + sqlite3_result_text(ctx, pCsr->zName, -1, SQLITE_TRANSIENT);
|
| + break;
|
| + case 1: /* path */
|
| + sqlite3_result_text(ctx, pCsr->zPath, -1, SQLITE_TRANSIENT);
|
| + break;
|
| + case 2: /* pageno */
|
| + sqlite3_result_int64(ctx, pCsr->iPageno);
|
| + break;
|
| + case 3: /* pagetype */
|
| + sqlite3_result_text(ctx, pCsr->zPagetype, -1, SQLITE_STATIC);
|
| + break;
|
| + case 4: /* ncell */
|
| + sqlite3_result_int(ctx, pCsr->nCell);
|
| + break;
|
| + case 5: /* payload */
|
| + sqlite3_result_int(ctx, pCsr->nPayload);
|
| + break;
|
| + case 6: /* unused */
|
| + sqlite3_result_int(ctx, pCsr->nUnused);
|
| + break;
|
| + case 7: /* mx_payload */
|
| + sqlite3_result_int(ctx, pCsr->nMxPayload);
|
| + break;
|
| + case 8: /* pgoffset */
|
| + sqlite3_result_int64(ctx, pCsr->iOffset);
|
| + break;
|
| + case 9: /* pgsize */
|
| + sqlite3_result_int(ctx, pCsr->szPage);
|
| + break;
|
| + default: { /* schema */
|
| + sqlite3 *db = sqlite3_context_db_handle(ctx);
|
| + int iDb = pCsr->iDb;
|
| + sqlite3_result_text(ctx, db->aDb[iDb].zName, -1, SQLITE_STATIC);
|
| + break;
|
| + }
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static int statRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
|
| + StatCursor *pCsr = (StatCursor *)pCursor;
|
| + *pRowid = pCsr->iPageno;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Invoke this routine to register the "dbstat" virtual table module
|
| +*/
|
| +SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3 *db){
|
| + static sqlite3_module dbstat_module = {
|
| + 0, /* iVersion */
|
| + statConnect, /* xCreate */
|
| + statConnect, /* xConnect */
|
| + statBestIndex, /* xBestIndex */
|
| + statDisconnect, /* xDisconnect */
|
| + statDisconnect, /* xDestroy */
|
| + statOpen, /* xOpen - open a cursor */
|
| + statClose, /* xClose - close a cursor */
|
| + statFilter, /* xFilter - configure scan constraints */
|
| + statNext, /* xNext - advance a cursor */
|
| + statEof, /* xEof - check for end of scan */
|
| + statColumn, /* xColumn - read data */
|
| + statRowid, /* xRowid - read data */
|
| + 0, /* xUpdate */
|
| + 0, /* xBegin */
|
| + 0, /* xSync */
|
| + 0, /* xCommit */
|
| + 0, /* xRollback */
|
| + 0, /* xFindMethod */
|
| + 0, /* xRename */
|
| + };
|
| + return sqlite3_create_module(db, "dbstat", &dbstat_module, 0);
|
| +}
|
| +#elif defined(SQLITE_ENABLE_DBSTAT_VTAB)
|
| +SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3 *db){ return SQLITE_OK; }
|
| +#endif /* SQLITE_ENABLE_DBSTAT_VTAB */
|
| +
|
| +/************** End of dbstat.c **********************************************/
|
| +/************** Begin file json1.c *******************************************/
|
| +/*
|
| +** 2015-08-12
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +** This SQLite extension implements JSON functions. The interface is
|
| +** modeled after MySQL JSON functions:
|
| +**
|
| +** https://dev.mysql.com/doc/refman/5.7/en/json.html
|
| +**
|
| +** For the time being, all JSON is stored as pure text. (We might add
|
| +** a JSONB type in the future which stores a binary encoding of JSON in
|
| +** a BLOB, but there is no support for JSONB in the current implementation.
|
| +** This implementation parses JSON text at 250 MB/s, so it is hard to see
|
| +** how JSONB might improve on that.)
|
| +*/
|
| +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1)
|
| +#if !defined(_SQLITEINT_H_)
|
| +/* #include "sqlite3ext.h" */
|
| +#endif
|
| +SQLITE_EXTENSION_INIT1
|
| +/* #include <assert.h> */
|
| +/* #include <string.h> */
|
| +/* #include <stdlib.h> */
|
| +/* #include <stdarg.h> */
|
| +
|
| +#define UNUSED_PARAM(X) (void)(X)
|
| +
|
| +#ifndef LARGEST_INT64
|
| +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32))
|
| +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64)
|
| +#endif
|
| +
|
| +/*
|
| +** Versions of isspace(), isalnum() and isdigit() to which it is safe
|
| +** to pass signed char values.
|
| +*/
|
| +#ifdef sqlite3Isdigit
|
| + /* Use the SQLite core versions if this routine is part of the
|
| + ** SQLite amalgamation */
|
| +# define safe_isdigit(x) sqlite3Isdigit(x)
|
| +# define safe_isalnum(x) sqlite3Isalnum(x)
|
| +#else
|
| + /* Use the standard library for separate compilation */
|
| +#include <ctype.h> /* amalgamator: keep */
|
| +# define safe_isdigit(x) isdigit((unsigned char)(x))
|
| +# define safe_isalnum(x) isalnum((unsigned char)(x))
|
| +#endif
|
| +
|
| +/*
|
| +** Growing our own isspace() routine this way is twice as fast as
|
| +** the library isspace() function, resulting in a 7% overall performance
|
| +** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os).
|
| +*/
|
| +static const char jsonIsSpace[] = {
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
| +};
|
| +#define safe_isspace(x) (jsonIsSpace[(unsigned char)x])
|
| +
|
| +#ifndef SQLITE_AMALGAMATION
|
| + /* Unsigned integer types. These are already defined in the sqliteInt.h,
|
| + ** but the definitions need to be repeated for separate compilation. */
|
| + typedef sqlite3_uint64 u64;
|
| + typedef unsigned int u32;
|
| + typedef unsigned char u8;
|
| +#endif
|
| +
|
| +/* Objects */
|
| +typedef struct JsonString JsonString;
|
| +typedef struct JsonNode JsonNode;
|
| +typedef struct JsonParse JsonParse;
|
| +
|
| +/* An instance of this object represents a JSON string
|
| +** under construction. Really, this is a generic string accumulator
|
| +** that can be and is used to create strings other than JSON.
|
| +*/
|
| +struct JsonString {
|
| + sqlite3_context *pCtx; /* Function context - put error messages here */
|
| + char *zBuf; /* Append JSON content here */
|
| + u64 nAlloc; /* Bytes of storage available in zBuf[] */
|
| + u64 nUsed; /* Bytes of zBuf[] currently used */
|
| + u8 bStatic; /* True if zBuf is static space */
|
| + u8 bErr; /* True if an error has been encountered */
|
| + char zSpace[100]; /* Initial static space */
|
| +};
|
| +
|
| +/* JSON type values
|
| +*/
|
| +#define JSON_NULL 0
|
| +#define JSON_TRUE 1
|
| +#define JSON_FALSE 2
|
| +#define JSON_INT 3
|
| +#define JSON_REAL 4
|
| +#define JSON_STRING 5
|
| +#define JSON_ARRAY 6
|
| +#define JSON_OBJECT 7
|
| +
|
| +/* The "subtype" set for JSON values */
|
| +#define JSON_SUBTYPE 74 /* Ascii for "J" */
|
| +
|
| +/*
|
| +** Names of the various JSON types:
|
| +*/
|
| +static const char * const jsonType[] = {
|
| + "null", "true", "false", "integer", "real", "text", "array", "object"
|
| +};
|
| +
|
| +/* Bit values for the JsonNode.jnFlag field
|
| +*/
|
| +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */
|
| +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */
|
| +#define JNODE_REMOVE 0x04 /* Do not output */
|
| +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */
|
| +#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */
|
| +#define JNODE_LABEL 0x20 /* Is a label of an object */
|
| +
|
| +
|
| +/* A single node of parsed JSON
|
| +*/
|
| +struct JsonNode {
|
| + u8 eType; /* One of the JSON_ type values */
|
| + u8 jnFlags; /* JNODE flags */
|
| + u8 iVal; /* Replacement value when JNODE_REPLACE */
|
| + u32 n; /* Bytes of content, or number of sub-nodes */
|
| + union {
|
| + const char *zJContent; /* Content for INT, REAL, and STRING */
|
| + u32 iAppend; /* More terms for ARRAY and OBJECT */
|
| + u32 iKey; /* Key for ARRAY objects in json_tree() */
|
| + } u;
|
| +};
|
| +
|
| +/* A completely parsed JSON string
|
| +*/
|
| +struct JsonParse {
|
| + u32 nNode; /* Number of slots of aNode[] used */
|
| + u32 nAlloc; /* Number of slots of aNode[] allocated */
|
| + JsonNode *aNode; /* Array of nodes containing the parse */
|
| + const char *zJson; /* Original JSON string */
|
| + u32 *aUp; /* Index of parent of each node */
|
| + u8 oom; /* Set to true if out of memory */
|
| + u8 nErr; /* Number of errors seen */
|
| +};
|
| +
|
| +/**************************************************************************
|
| +** Utility routines for dealing with JsonString objects
|
| +**************************************************************************/
|
| +
|
| +/* Set the JsonString object to an empty string
|
| +*/
|
| +static void jsonZero(JsonString *p){
|
| + p->zBuf = p->zSpace;
|
| + p->nAlloc = sizeof(p->zSpace);
|
| + p->nUsed = 0;
|
| + p->bStatic = 1;
|
| +}
|
| +
|
| +/* Initialize the JsonString object
|
| +*/
|
| +static void jsonInit(JsonString *p, sqlite3_context *pCtx){
|
| + p->pCtx = pCtx;
|
| + p->bErr = 0;
|
| + jsonZero(p);
|
| +}
|
| +
|
| +
|
| +/* Free all allocated memory and reset the JsonString object back to its
|
| +** initial state.
|
| +*/
|
| +static void jsonReset(JsonString *p){
|
| + if( !p->bStatic ) sqlite3_free(p->zBuf);
|
| + jsonZero(p);
|
| +}
|
| +
|
| +
|
| +/* Report an out-of-memory (OOM) condition
|
| +*/
|
| +static void jsonOom(JsonString *p){
|
| + p->bErr = 1;
|
| + sqlite3_result_error_nomem(p->pCtx);
|
| + jsonReset(p);
|
| +}
|
| +
|
| +/* Enlarge pJson->zBuf so that it can hold at least N more bytes.
|
| +** Return zero on success. Return non-zero on an OOM error
|
| +*/
|
| +static int jsonGrow(JsonString *p, u32 N){
|
| + u64 nTotal = N<p->nAlloc ? p->nAlloc*2 : p->nAlloc+N+10;
|
| + char *zNew;
|
| + if( p->bStatic ){
|
| + if( p->bErr ) return 1;
|
| + zNew = sqlite3_malloc64(nTotal);
|
| + if( zNew==0 ){
|
| + jsonOom(p);
|
| + return SQLITE_NOMEM;
|
| + }
|
| + memcpy(zNew, p->zBuf, (size_t)p->nUsed);
|
| + p->zBuf = zNew;
|
| + p->bStatic = 0;
|
| + }else{
|
| + zNew = sqlite3_realloc64(p->zBuf, nTotal);
|
| + if( zNew==0 ){
|
| + jsonOom(p);
|
| + return SQLITE_NOMEM;
|
| + }
|
| + p->zBuf = zNew;
|
| + }
|
| + p->nAlloc = nTotal;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Append N bytes from zIn onto the end of the JsonString string.
|
| +*/
|
| +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){
|
| + if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return;
|
| + memcpy(p->zBuf+p->nUsed, zIn, N);
|
| + p->nUsed += N;
|
| +}
|
| +
|
| +/* Append formatted text (not to exceed N bytes) to the JsonString.
|
| +*/
|
| +static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){
|
| + va_list ap;
|
| + if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return;
|
| + va_start(ap, zFormat);
|
| + sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap);
|
| + va_end(ap);
|
| + p->nUsed += (int)strlen(p->zBuf+p->nUsed);
|
| +}
|
| +
|
| +/* Append a single character
|
| +*/
|
| +static void jsonAppendChar(JsonString *p, char c){
|
| + if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return;
|
| + p->zBuf[p->nUsed++] = c;
|
| +}
|
| +
|
| +/* Append a comma separator to the output buffer, if the previous
|
| +** character is not '[' or '{'.
|
| +*/
|
| +static void jsonAppendSeparator(JsonString *p){
|
| + char c;
|
| + if( p->nUsed==0 ) return;
|
| + c = p->zBuf[p->nUsed-1];
|
| + if( c!='[' && c!='{' ) jsonAppendChar(p, ',');
|
| +}
|
| +
|
| +/* Append the N-byte string in zIn to the end of the JsonString string
|
| +** under construction. Enclose the string in "..." and escape
|
| +** any double-quotes or backslash characters contained within the
|
| +** string.
|
| +*/
|
| +static void jsonAppendString(JsonString *p, const char *zIn, u32 N){
|
| + u32 i;
|
| + if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return;
|
| + p->zBuf[p->nUsed++] = '"';
|
| + for(i=0; i<N; i++){
|
| + char c = zIn[i];
|
| + if( c=='"' || c=='\\' ){
|
| + if( (p->nUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return;
|
| + p->zBuf[p->nUsed++] = '\\';
|
| + }
|
| + p->zBuf[p->nUsed++] = c;
|
| + }
|
| + p->zBuf[p->nUsed++] = '"';
|
| + assert( p->nUsed<p->nAlloc );
|
| +}
|
| +
|
| +/*
|
| +** Append a function parameter value to the JSON string under
|
| +** construction.
|
| +*/
|
| +static void jsonAppendValue(
|
| + JsonString *p, /* Append to this JSON string */
|
| + sqlite3_value *pValue /* Value to append */
|
| +){
|
| + switch( sqlite3_value_type(pValue) ){
|
| + case SQLITE_NULL: {
|
| + jsonAppendRaw(p, "null", 4);
|
| + break;
|
| + }
|
| + case SQLITE_INTEGER:
|
| + case SQLITE_FLOAT: {
|
| + const char *z = (const char*)sqlite3_value_text(pValue);
|
| + u32 n = (u32)sqlite3_value_bytes(pValue);
|
| + jsonAppendRaw(p, z, n);
|
| + break;
|
| + }
|
| + case SQLITE_TEXT: {
|
| + const char *z = (const char*)sqlite3_value_text(pValue);
|
| + u32 n = (u32)sqlite3_value_bytes(pValue);
|
| + if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){
|
| + jsonAppendRaw(p, z, n);
|
| + }else{
|
| + jsonAppendString(p, z, n);
|
| + }
|
| + break;
|
| + }
|
| + default: {
|
| + if( p->bErr==0 ){
|
| + sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1);
|
| + p->bErr = 1;
|
| + jsonReset(p);
|
| + }
|
| + break;
|
| + }
|
| + }
|
| +}
|
| +
|
| +
|
| +/* Make the JSON in p the result of the SQL function.
|
| +*/
|
| +static void jsonResult(JsonString *p){
|
| + if( p->bErr==0 ){
|
| + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed,
|
| + p->bStatic ? SQLITE_TRANSIENT : sqlite3_free,
|
| + SQLITE_UTF8);
|
| + jsonZero(p);
|
| + }
|
| + assert( p->bStatic );
|
| +}
|
| +
|
| +/**************************************************************************
|
| +** Utility routines for dealing with JsonNode and JsonParse objects
|
| +**************************************************************************/
|
| +
|
| +/*
|
| +** Return the number of consecutive JsonNode slots need to represent
|
| +** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and
|
| +** OBJECT types, the number might be larger.
|
| +**
|
| +** Appended elements are not counted. The value returned is the number
|
| +** by which the JsonNode counter should increment in order to go to the
|
| +** next peer value.
|
| +*/
|
| +static u32 jsonNodeSize(JsonNode *pNode){
|
| + return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1;
|
| +}
|
| +
|
| +/*
|
| +** Reclaim all memory allocated by a JsonParse object. But do not
|
| +** delete the JsonParse object itself.
|
| +*/
|
| +static void jsonParseReset(JsonParse *pParse){
|
| + sqlite3_free(pParse->aNode);
|
| + pParse->aNode = 0;
|
| + pParse->nNode = 0;
|
| + pParse->nAlloc = 0;
|
| + sqlite3_free(pParse->aUp);
|
| + pParse->aUp = 0;
|
| +}
|
| +
|
| +/*
|
| +** Convert the JsonNode pNode into a pure JSON string and
|
| +** append to pOut. Subsubstructure is also included. Return
|
| +** the number of JsonNode objects that are encoded.
|
| +*/
|
| +static void jsonRenderNode(
|
| + JsonNode *pNode, /* The node to render */
|
| + JsonString *pOut, /* Write JSON here */
|
| + sqlite3_value **aReplace /* Replacement values */
|
| +){
|
| + switch( pNode->eType ){
|
| + default: {
|
| + assert( pNode->eType==JSON_NULL );
|
| + jsonAppendRaw(pOut, "null", 4);
|
| + break;
|
| + }
|
| + case JSON_TRUE: {
|
| + jsonAppendRaw(pOut, "true", 4);
|
| + break;
|
| + }
|
| + case JSON_FALSE: {
|
| + jsonAppendRaw(pOut, "false", 5);
|
| + break;
|
| + }
|
| + case JSON_STRING: {
|
| + if( pNode->jnFlags & JNODE_RAW ){
|
| + jsonAppendString(pOut, pNode->u.zJContent, pNode->n);
|
| + break;
|
| + }
|
| + /* Fall through into the next case */
|
| + }
|
| + case JSON_REAL:
|
| + case JSON_INT: {
|
| + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n);
|
| + break;
|
| + }
|
| + case JSON_ARRAY: {
|
| + u32 j = 1;
|
| + jsonAppendChar(pOut, '[');
|
| + for(;;){
|
| + while( j<=pNode->n ){
|
| + if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){
|
| + if( pNode[j].jnFlags & JNODE_REPLACE ){
|
| + jsonAppendSeparator(pOut);
|
| + jsonAppendValue(pOut, aReplace[pNode[j].iVal]);
|
| + }
|
| + }else{
|
| + jsonAppendSeparator(pOut);
|
| + jsonRenderNode(&pNode[j], pOut, aReplace);
|
| + }
|
| + j += jsonNodeSize(&pNode[j]);
|
| + }
|
| + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
|
| + pNode = &pNode[pNode->u.iAppend];
|
| + j = 1;
|
| + }
|
| + jsonAppendChar(pOut, ']');
|
| + break;
|
| + }
|
| + case JSON_OBJECT: {
|
| + u32 j = 1;
|
| + jsonAppendChar(pOut, '{');
|
| + for(;;){
|
| + while( j<=pNode->n ){
|
| + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){
|
| + jsonAppendSeparator(pOut);
|
| + jsonRenderNode(&pNode[j], pOut, aReplace);
|
| + jsonAppendChar(pOut, ':');
|
| + if( pNode[j+1].jnFlags & JNODE_REPLACE ){
|
| + jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]);
|
| + }else{
|
| + jsonRenderNode(&pNode[j+1], pOut, aReplace);
|
| + }
|
| + }
|
| + j += 1 + jsonNodeSize(&pNode[j+1]);
|
| + }
|
| + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break;
|
| + pNode = &pNode[pNode->u.iAppend];
|
| + j = 1;
|
| + }
|
| + jsonAppendChar(pOut, '}');
|
| + break;
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Return a JsonNode and all its descendents as a JSON string.
|
| +*/
|
| +static void jsonReturnJson(
|
| + JsonNode *pNode, /* Node to return */
|
| + sqlite3_context *pCtx, /* Return value for this function */
|
| + sqlite3_value **aReplace /* Array of replacement values */
|
| +){
|
| + JsonString s;
|
| + jsonInit(&s, pCtx);
|
| + jsonRenderNode(pNode, &s, aReplace);
|
| + jsonResult(&s);
|
| + sqlite3_result_subtype(pCtx, JSON_SUBTYPE);
|
| +}
|
| +
|
| +/*
|
| +** Make the JsonNode the return value of the function.
|
| +*/
|
| +static void jsonReturn(
|
| + JsonNode *pNode, /* Node to return */
|
| + sqlite3_context *pCtx, /* Return value for this function */
|
| + sqlite3_value **aReplace /* Array of replacement values */
|
| +){
|
| + switch( pNode->eType ){
|
| + default: {
|
| + assert( pNode->eType==JSON_NULL );
|
| + sqlite3_result_null(pCtx);
|
| + break;
|
| + }
|
| + case JSON_TRUE: {
|
| + sqlite3_result_int(pCtx, 1);
|
| + break;
|
| + }
|
| + case JSON_FALSE: {
|
| + sqlite3_result_int(pCtx, 0);
|
| + break;
|
| + }
|
| + case JSON_INT: {
|
| + sqlite3_int64 i = 0;
|
| + const char *z = pNode->u.zJContent;
|
| + if( z[0]=='-' ){ z++; }
|
| + while( z[0]>='0' && z[0]<='9' ){
|
| + unsigned v = *(z++) - '0';
|
| + if( i>=LARGEST_INT64/10 ){
|
| + if( i>LARGEST_INT64/10 ) goto int_as_real;
|
| + if( z[0]>='0' && z[0]<='9' ) goto int_as_real;
|
| + if( v==9 ) goto int_as_real;
|
| + if( v==8 ){
|
| + if( pNode->u.zJContent[0]=='-' ){
|
| + sqlite3_result_int64(pCtx, SMALLEST_INT64);
|
| + goto int_done;
|
| + }else{
|
| + goto int_as_real;
|
| + }
|
| + }
|
| + }
|
| + i = i*10 + v;
|
| + }
|
| + if( pNode->u.zJContent[0]=='-' ){ i = -i; }
|
| + sqlite3_result_int64(pCtx, i);
|
| + int_done:
|
| + break;
|
| + int_as_real: /* fall through to real */;
|
| + }
|
| + case JSON_REAL: {
|
| + double r;
|
| +#ifdef SQLITE_AMALGAMATION
|
| + const char *z = pNode->u.zJContent;
|
| + sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8);
|
| +#else
|
| + r = strtod(pNode->u.zJContent, 0);
|
| +#endif
|
| + sqlite3_result_double(pCtx, r);
|
| + break;
|
| + }
|
| + case JSON_STRING: {
|
| +#if 0 /* Never happens because JNODE_RAW is only set by json_set(),
|
| + ** json_insert() and json_replace() and those routines do not
|
| + ** call jsonReturn() */
|
| + if( pNode->jnFlags & JNODE_RAW ){
|
| + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n,
|
| + SQLITE_TRANSIENT);
|
| + }else
|
| +#endif
|
| + assert( (pNode->jnFlags & JNODE_RAW)==0 );
|
| + if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){
|
| + /* JSON formatted without any backslash-escapes */
|
| + sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2,
|
| + SQLITE_TRANSIENT);
|
| + }else{
|
| + /* Translate JSON formatted string into raw text */
|
| + u32 i;
|
| + u32 n = pNode->n;
|
| + const char *z = pNode->u.zJContent;
|
| + char *zOut;
|
| + u32 j;
|
| + zOut = sqlite3_malloc( n+1 );
|
| + if( zOut==0 ){
|
| + sqlite3_result_error_nomem(pCtx);
|
| + break;
|
| + }
|
| + for(i=1, j=0; i<n-1; i++){
|
| + char c = z[i];
|
| + if( c!='\\' ){
|
| + zOut[j++] = c;
|
| + }else{
|
| + c = z[++i];
|
| + if( c=='u' ){
|
| + u32 v = 0, k;
|
| + for(k=0; k<4 && i<n-2; i++, k++){
|
| + c = z[i+1];
|
| + if( c>='0' && c<='9' ) v = v*16 + c - '0';
|
| + else if( c>='A' && c<='F' ) v = v*16 + c - 'A' + 10;
|
| + else if( c>='a' && c<='f' ) v = v*16 + c - 'a' + 10;
|
| + else break;
|
| + }
|
| + if( v==0 ) break;
|
| + if( v<=0x7f ){
|
| + zOut[j++] = (char)v;
|
| + }else if( v<=0x7ff ){
|
| + zOut[j++] = (char)(0xc0 | (v>>6));
|
| + zOut[j++] = 0x80 | (v&0x3f);
|
| + }else{
|
| + zOut[j++] = (char)(0xe0 | (v>>12));
|
| + zOut[j++] = 0x80 | ((v>>6)&0x3f);
|
| + zOut[j++] = 0x80 | (v&0x3f);
|
| + }
|
| + }else{
|
| + if( c=='b' ){
|
| + c = '\b';
|
| + }else if( c=='f' ){
|
| + c = '\f';
|
| + }else if( c=='n' ){
|
| + c = '\n';
|
| + }else if( c=='r' ){
|
| + c = '\r';
|
| + }else if( c=='t' ){
|
| + c = '\t';
|
| + }
|
| + zOut[j++] = c;
|
| + }
|
| + }
|
| + }
|
| + zOut[j] = 0;
|
| + sqlite3_result_text(pCtx, zOut, j, sqlite3_free);
|
| + }
|
| + break;
|
| + }
|
| + case JSON_ARRAY:
|
| + case JSON_OBJECT: {
|
| + jsonReturnJson(pNode, pCtx, aReplace);
|
| + break;
|
| + }
|
| + }
|
| +}
|
| +
|
| +/* Forward reference */
|
| +static int jsonParseAddNode(JsonParse*,u32,u32,const char*);
|
| +
|
| +/*
|
| +** A macro to hint to the compiler that a function should not be
|
| +** inlined.
|
| +*/
|
| +#if defined(__GNUC__)
|
| +# define JSON_NOINLINE __attribute__((noinline))
|
| +#elif defined(_MSC_VER) && _MSC_VER>=1310
|
| +# define JSON_NOINLINE __declspec(noinline)
|
| +#else
|
| +# define JSON_NOINLINE
|
| +#endif
|
| +
|
| +
|
| +static JSON_NOINLINE int jsonParseAddNodeExpand(
|
| + JsonParse *pParse, /* Append the node to this object */
|
| + u32 eType, /* Node type */
|
| + u32 n, /* Content size or sub-node count */
|
| + const char *zContent /* Content */
|
| +){
|
| + u32 nNew;
|
| + JsonNode *pNew;
|
| + assert( pParse->nNode>=pParse->nAlloc );
|
| + if( pParse->oom ) return -1;
|
| + nNew = pParse->nAlloc*2 + 10;
|
| + pNew = sqlite3_realloc(pParse->aNode, sizeof(JsonNode)*nNew);
|
| + if( pNew==0 ){
|
| + pParse->oom = 1;
|
| + return -1;
|
| + }
|
| + pParse->nAlloc = nNew;
|
| + pParse->aNode = pNew;
|
| + assert( pParse->nNode<pParse->nAlloc );
|
| + return jsonParseAddNode(pParse, eType, n, zContent);
|
| +}
|
| +
|
| +/*
|
| +** Create a new JsonNode instance based on the arguments and append that
|
| +** instance to the JsonParse. Return the index in pParse->aNode[] of the
|
| +** new node, or -1 if a memory allocation fails.
|
| +*/
|
| +static int jsonParseAddNode(
|
| + JsonParse *pParse, /* Append the node to this object */
|
| + u32 eType, /* Node type */
|
| + u32 n, /* Content size or sub-node count */
|
| + const char *zContent /* Content */
|
| +){
|
| + JsonNode *p;
|
| + if( pParse->nNode>=pParse->nAlloc ){
|
| + return jsonParseAddNodeExpand(pParse, eType, n, zContent);
|
| + }
|
| + p = &pParse->aNode[pParse->nNode];
|
| + p->eType = (u8)eType;
|
| + p->jnFlags = 0;
|
| + p->iVal = 0;
|
| + p->n = n;
|
| + p->u.zJContent = zContent;
|
| + return pParse->nNode++;
|
| +}
|
| +
|
| +/*
|
| +** Parse a single JSON value which begins at pParse->zJson[i]. Return the
|
| +** index of the first character past the end of the value parsed.
|
| +**
|
| +** Return negative for a syntax error. Special cases: return -2 if the
|
| +** first non-whitespace character is '}' and return -3 if the first
|
| +** non-whitespace character is ']'.
|
| +*/
|
| +static int jsonParseValue(JsonParse *pParse, u32 i){
|
| + char c;
|
| + u32 j;
|
| + int iThis;
|
| + int x;
|
| + JsonNode *pNode;
|
| + while( safe_isspace(pParse->zJson[i]) ){ i++; }
|
| + if( (c = pParse->zJson[i])=='{' ){
|
| + /* Parse object */
|
| + iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0);
|
| + if( iThis<0 ) return -1;
|
| + for(j=i+1;;j++){
|
| + while( safe_isspace(pParse->zJson[j]) ){ j++; }
|
| + x = jsonParseValue(pParse, j);
|
| + if( x<0 ){
|
| + if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1;
|
| + return -1;
|
| + }
|
| + if( pParse->oom ) return -1;
|
| + pNode = &pParse->aNode[pParse->nNode-1];
|
| + if( pNode->eType!=JSON_STRING ) return -1;
|
| + pNode->jnFlags |= JNODE_LABEL;
|
| + j = x;
|
| + while( safe_isspace(pParse->zJson[j]) ){ j++; }
|
| + if( pParse->zJson[j]!=':' ) return -1;
|
| + j++;
|
| + x = jsonParseValue(pParse, j);
|
| + if( x<0 ) return -1;
|
| + j = x;
|
| + while( safe_isspace(pParse->zJson[j]) ){ j++; }
|
| + c = pParse->zJson[j];
|
| + if( c==',' ) continue;
|
| + if( c!='}' ) return -1;
|
| + break;
|
| + }
|
| + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1;
|
| + return j+1;
|
| + }else if( c=='[' ){
|
| + /* Parse array */
|
| + iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0);
|
| + if( iThis<0 ) return -1;
|
| + for(j=i+1;;j++){
|
| + while( safe_isspace(pParse->zJson[j]) ){ j++; }
|
| + x = jsonParseValue(pParse, j);
|
| + if( x<0 ){
|
| + if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1;
|
| + return -1;
|
| + }
|
| + j = x;
|
| + while( safe_isspace(pParse->zJson[j]) ){ j++; }
|
| + c = pParse->zJson[j];
|
| + if( c==',' ) continue;
|
| + if( c!=']' ) return -1;
|
| + break;
|
| + }
|
| + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1;
|
| + return j+1;
|
| + }else if( c=='"' ){
|
| + /* Parse string */
|
| + u8 jnFlags = 0;
|
| + j = i+1;
|
| + for(;;){
|
| + c = pParse->zJson[j];
|
| + if( c==0 ) return -1;
|
| + if( c=='\\' ){
|
| + c = pParse->zJson[++j];
|
| + if( c==0 ) return -1;
|
| + jnFlags = JNODE_ESCAPE;
|
| + }else if( c=='"' ){
|
| + break;
|
| + }
|
| + j++;
|
| + }
|
| + jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]);
|
| + if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags;
|
| + return j+1;
|
| + }else if( c=='n'
|
| + && strncmp(pParse->zJson+i,"null",4)==0
|
| + && !safe_isalnum(pParse->zJson[i+4]) ){
|
| + jsonParseAddNode(pParse, JSON_NULL, 0, 0);
|
| + return i+4;
|
| + }else if( c=='t'
|
| + && strncmp(pParse->zJson+i,"true",4)==0
|
| + && !safe_isalnum(pParse->zJson[i+4]) ){
|
| + jsonParseAddNode(pParse, JSON_TRUE, 0, 0);
|
| + return i+4;
|
| + }else if( c=='f'
|
| + && strncmp(pParse->zJson+i,"false",5)==0
|
| + && !safe_isalnum(pParse->zJson[i+5]) ){
|
| + jsonParseAddNode(pParse, JSON_FALSE, 0, 0);
|
| + return i+5;
|
| + }else if( c=='-' || (c>='0' && c<='9') ){
|
| + /* Parse number */
|
| + u8 seenDP = 0;
|
| + u8 seenE = 0;
|
| + j = i+1;
|
| + for(;; j++){
|
| + c = pParse->zJson[j];
|
| + if( c>='0' && c<='9' ) continue;
|
| + if( c=='.' ){
|
| + if( pParse->zJson[j-1]=='-' ) return -1;
|
| + if( seenDP ) return -1;
|
| + seenDP = 1;
|
| + continue;
|
| + }
|
| + if( c=='e' || c=='E' ){
|
| + if( pParse->zJson[j-1]<'0' ) return -1;
|
| + if( seenE ) return -1;
|
| + seenDP = seenE = 1;
|
| + c = pParse->zJson[j+1];
|
| + if( c=='+' || c=='-' ){
|
| + j++;
|
| + c = pParse->zJson[j+1];
|
| + }
|
| + if( c<'0' || c>'9' ) return -1;
|
| + continue;
|
| + }
|
| + break;
|
| + }
|
| + if( pParse->zJson[j-1]<'0' ) return -1;
|
| + jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT,
|
| + j - i, &pParse->zJson[i]);
|
| + return j;
|
| + }else if( c=='}' ){
|
| + return -2; /* End of {...} */
|
| + }else if( c==']' ){
|
| + return -3; /* End of [...] */
|
| + }else if( c==0 ){
|
| + return 0; /* End of file */
|
| + }else{
|
| + return -1; /* Syntax error */
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Parse a complete JSON string. Return 0 on success or non-zero if there
|
| +** are any errors. If an error occurs, free all memory associated with
|
| +** pParse.
|
| +**
|
| +** pParse is uninitialized when this routine is called.
|
| +*/
|
| +static int jsonParse(
|
| + JsonParse *pParse, /* Initialize and fill this JsonParse object */
|
| + sqlite3_context *pCtx, /* Report errors here */
|
| + const char *zJson /* Input JSON text to be parsed */
|
| +){
|
| + int i;
|
| + memset(pParse, 0, sizeof(*pParse));
|
| + if( zJson==0 ) return 1;
|
| + pParse->zJson = zJson;
|
| + i = jsonParseValue(pParse, 0);
|
| + if( pParse->oom ) i = -1;
|
| + if( i>0 ){
|
| + while( safe_isspace(zJson[i]) ) i++;
|
| + if( zJson[i] ) i = -1;
|
| + }
|
| + if( i<=0 ){
|
| + if( pCtx!=0 ){
|
| + if( pParse->oom ){
|
| + sqlite3_result_error_nomem(pCtx);
|
| + }else{
|
| + sqlite3_result_error(pCtx, "malformed JSON", -1);
|
| + }
|
| + }
|
| + jsonParseReset(pParse);
|
| + return 1;
|
| + }
|
| + return 0;
|
| +}
|
| +
|
| +/* Mark node i of pParse as being a child of iParent. Call recursively
|
| +** to fill in all the descendants of node i.
|
| +*/
|
| +static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){
|
| + JsonNode *pNode = &pParse->aNode[i];
|
| + u32 j;
|
| + pParse->aUp[i] = iParent;
|
| + switch( pNode->eType ){
|
| + case JSON_ARRAY: {
|
| + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){
|
| + jsonParseFillInParentage(pParse, i+j, i);
|
| + }
|
| + break;
|
| + }
|
| + case JSON_OBJECT: {
|
| + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){
|
| + pParse->aUp[i+j] = i;
|
| + jsonParseFillInParentage(pParse, i+j+1, i);
|
| + }
|
| + break;
|
| + }
|
| + default: {
|
| + break;
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Compute the parentage of all nodes in a completed parse.
|
| +*/
|
| +static int jsonParseFindParents(JsonParse *pParse){
|
| + u32 *aUp;
|
| + assert( pParse->aUp==0 );
|
| + aUp = pParse->aUp = sqlite3_malloc( sizeof(u32)*pParse->nNode );
|
| + if( aUp==0 ){
|
| + pParse->oom = 1;
|
| + return SQLITE_NOMEM;
|
| + }
|
| + jsonParseFillInParentage(pParse, 0, 0);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Compare the OBJECT label at pNode against zKey,nKey. Return true on
|
| +** a match.
|
| +*/
|
| +static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){
|
| + if( pNode->jnFlags & JNODE_RAW ){
|
| + if( pNode->n!=nKey ) return 0;
|
| + return strncmp(pNode->u.zJContent, zKey, nKey)==0;
|
| + }else{
|
| + if( pNode->n!=nKey+2 ) return 0;
|
| + return strncmp(pNode->u.zJContent+1, zKey, nKey)==0;
|
| + }
|
| +}
|
| +
|
| +/* forward declaration */
|
| +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**);
|
| +
|
| +/*
|
| +** Search along zPath to find the node specified. Return a pointer
|
| +** to that node, or NULL if zPath is malformed or if there is no such
|
| +** node.
|
| +**
|
| +** If pApnd!=0, then try to append new nodes to complete zPath if it is
|
| +** possible to do so and if no existing node corresponds to zPath. If
|
| +** new nodes are appended *pApnd is set to 1.
|
| +*/
|
| +static JsonNode *jsonLookupStep(
|
| + JsonParse *pParse, /* The JSON to search */
|
| + u32 iRoot, /* Begin the search at this node */
|
| + const char *zPath, /* The path to search */
|
| + int *pApnd, /* Append nodes to complete path if not NULL */
|
| + const char **pzErr /* Make *pzErr point to any syntax error in zPath */
|
| +){
|
| + u32 i, j, nKey;
|
| + const char *zKey;
|
| + JsonNode *pRoot = &pParse->aNode[iRoot];
|
| + if( zPath[0]==0 ) return pRoot;
|
| + if( zPath[0]=='.' ){
|
| + if( pRoot->eType!=JSON_OBJECT ) return 0;
|
| + zPath++;
|
| + if( zPath[0]=='"' ){
|
| + zKey = zPath + 1;
|
| + for(i=1; zPath[i] && zPath[i]!='"'; i++){}
|
| + nKey = i-1;
|
| + if( zPath[i] ){
|
| + i++;
|
| + }else{
|
| + *pzErr = zPath;
|
| + return 0;
|
| + }
|
| + }else{
|
| + zKey = zPath;
|
| + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){}
|
| + nKey = i;
|
| + }
|
| + if( nKey==0 ){
|
| + *pzErr = zPath;
|
| + return 0;
|
| + }
|
| + j = 1;
|
| + for(;;){
|
| + while( j<=pRoot->n ){
|
| + if( jsonLabelCompare(pRoot+j, zKey, nKey) ){
|
| + return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr);
|
| + }
|
| + j++;
|
| + j += jsonNodeSize(&pRoot[j]);
|
| + }
|
| + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break;
|
| + iRoot += pRoot->u.iAppend;
|
| + pRoot = &pParse->aNode[iRoot];
|
| + j = 1;
|
| + }
|
| + if( pApnd ){
|
| + u32 iStart, iLabel;
|
| + JsonNode *pNode;
|
| + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0);
|
| + iLabel = jsonParseAddNode(pParse, JSON_STRING, i, zPath);
|
| + zPath += i;
|
| + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr);
|
| + if( pParse->oom ) return 0;
|
| + if( pNode ){
|
| + pRoot = &pParse->aNode[iRoot];
|
| + pRoot->u.iAppend = iStart - iRoot;
|
| + pRoot->jnFlags |= JNODE_APPEND;
|
| + pParse->aNode[iLabel].jnFlags |= JNODE_RAW;
|
| + }
|
| + return pNode;
|
| + }
|
| + }else if( zPath[0]=='[' && safe_isdigit(zPath[1]) ){
|
| + if( pRoot->eType!=JSON_ARRAY ) return 0;
|
| + i = 0;
|
| + j = 1;
|
| + while( safe_isdigit(zPath[j]) ){
|
| + i = i*10 + zPath[j] - '0';
|
| + j++;
|
| + }
|
| + if( zPath[j]!=']' ){
|
| + *pzErr = zPath;
|
| + return 0;
|
| + }
|
| + zPath += j + 1;
|
| + j = 1;
|
| + for(;;){
|
| + while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){
|
| + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--;
|
| + j += jsonNodeSize(&pRoot[j]);
|
| + }
|
| + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break;
|
| + iRoot += pRoot->u.iAppend;
|
| + pRoot = &pParse->aNode[iRoot];
|
| + j = 1;
|
| + }
|
| + if( j<=pRoot->n ){
|
| + return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr);
|
| + }
|
| + if( i==0 && pApnd ){
|
| + u32 iStart;
|
| + JsonNode *pNode;
|
| + iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0);
|
| + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr);
|
| + if( pParse->oom ) return 0;
|
| + if( pNode ){
|
| + pRoot = &pParse->aNode[iRoot];
|
| + pRoot->u.iAppend = iStart - iRoot;
|
| + pRoot->jnFlags |= JNODE_APPEND;
|
| + }
|
| + return pNode;
|
| + }
|
| + }else{
|
| + *pzErr = zPath;
|
| + }
|
| + return 0;
|
| +}
|
| +
|
| +/*
|
| +** Append content to pParse that will complete zPath. Return a pointer
|
| +** to the inserted node, or return NULL if the append fails.
|
| +*/
|
| +static JsonNode *jsonLookupAppend(
|
| + JsonParse *pParse, /* Append content to the JSON parse */
|
| + const char *zPath, /* Description of content to append */
|
| + int *pApnd, /* Set this flag to 1 */
|
| + const char **pzErr /* Make this point to any syntax error */
|
| +){
|
| + *pApnd = 1;
|
| + if( zPath[0]==0 ){
|
| + jsonParseAddNode(pParse, JSON_NULL, 0, 0);
|
| + return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1];
|
| + }
|
| + if( zPath[0]=='.' ){
|
| + jsonParseAddNode(pParse, JSON_OBJECT, 0, 0);
|
| + }else if( strncmp(zPath,"[0]",3)==0 ){
|
| + jsonParseAddNode(pParse, JSON_ARRAY, 0, 0);
|
| + }else{
|
| + return 0;
|
| + }
|
| + if( pParse->oom ) return 0;
|
| + return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr);
|
| +}
|
| +
|
| +/*
|
| +** Return the text of a syntax error message on a JSON path. Space is
|
| +** obtained from sqlite3_malloc().
|
| +*/
|
| +static char *jsonPathSyntaxError(const char *zErr){
|
| + return sqlite3_mprintf("JSON path error near '%q'", zErr);
|
| +}
|
| +
|
| +/*
|
| +** Do a node lookup using zPath. Return a pointer to the node on success.
|
| +** Return NULL if not found or if there is an error.
|
| +**
|
| +** On an error, write an error message into pCtx and increment the
|
| +** pParse->nErr counter.
|
| +**
|
| +** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if
|
| +** nodes are appended.
|
| +*/
|
| +static JsonNode *jsonLookup(
|
| + JsonParse *pParse, /* The JSON to search */
|
| + const char *zPath, /* The path to search */
|
| + int *pApnd, /* Append nodes to complete path if not NULL */
|
| + sqlite3_context *pCtx /* Report errors here, if not NULL */
|
| +){
|
| + const char *zErr = 0;
|
| + JsonNode *pNode = 0;
|
| + char *zMsg;
|
| +
|
| + if( zPath==0 ) return 0;
|
| + if( zPath[0]!='$' ){
|
| + zErr = zPath;
|
| + goto lookup_err;
|
| + }
|
| + zPath++;
|
| + pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr);
|
| + if( zErr==0 ) return pNode;
|
| +
|
| +lookup_err:
|
| + pParse->nErr++;
|
| + assert( zErr!=0 && pCtx!=0 );
|
| + zMsg = jsonPathSyntaxError(zErr);
|
| + if( zMsg ){
|
| + sqlite3_result_error(pCtx, zMsg, -1);
|
| + sqlite3_free(zMsg);
|
| + }else{
|
| + sqlite3_result_error_nomem(pCtx);
|
| + }
|
| + return 0;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Report the wrong number of arguments for json_insert(), json_replace()
|
| +** or json_set().
|
| +*/
|
| +static void jsonWrongNumArgs(
|
| + sqlite3_context *pCtx,
|
| + const char *zFuncName
|
| +){
|
| + char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments",
|
| + zFuncName);
|
| + sqlite3_result_error(pCtx, zMsg, -1);
|
| + sqlite3_free(zMsg);
|
| +}
|
| +
|
| +
|
| +/****************************************************************************
|
| +** SQL functions used for testing and debugging
|
| +****************************************************************************/
|
| +
|
| +#ifdef SQLITE_DEBUG
|
| +/*
|
| +** The json_parse(JSON) function returns a string which describes
|
| +** a parse of the JSON provided. Or it returns NULL if JSON is not
|
| +** well-formed.
|
| +*/
|
| +static void jsonParseFunc(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + JsonString s; /* Output string - not real JSON */
|
| + JsonParse x; /* The parse */
|
| + u32 i;
|
| +
|
| + assert( argc==1 );
|
| + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
|
| + jsonParseFindParents(&x);
|
| + jsonInit(&s, ctx);
|
| + for(i=0; i<x.nNode; i++){
|
| + const char *zType;
|
| + if( x.aNode[i].jnFlags & JNODE_LABEL ){
|
| + assert( x.aNode[i].eType==JSON_STRING );
|
| + zType = "label";
|
| + }else{
|
| + zType = jsonType[x.aNode[i].eType];
|
| + }
|
| + jsonPrintf(100, &s,"node %3u: %7s n=%-4d up=%-4d",
|
| + i, zType, x.aNode[i].n, x.aUp[i]);
|
| + if( x.aNode[i].u.zJContent!=0 ){
|
| + jsonAppendRaw(&s, " ", 1);
|
| + jsonAppendRaw(&s, x.aNode[i].u.zJContent, x.aNode[i].n);
|
| + }
|
| + jsonAppendRaw(&s, "\n", 1);
|
| + }
|
| + jsonParseReset(&x);
|
| + jsonResult(&s);
|
| +}
|
| +
|
| +/*
|
| +** The json_test1(JSON) function return true (1) if the input is JSON
|
| +** text generated by another json function. It returns (0) if the input
|
| +** is not known to be JSON.
|
| +*/
|
| +static void jsonTest1Func(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + UNUSED_PARAM(argc);
|
| + sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE);
|
| +}
|
| +#endif /* SQLITE_DEBUG */
|
| +
|
| +/****************************************************************************
|
| +** Scalar SQL function implementations
|
| +****************************************************************************/
|
| +
|
| +/*
|
| +** Implementation of the json_array(VALUE,...) function. Return a JSON
|
| +** array that contains all values given in arguments. Or if any argument
|
| +** is a BLOB, throw an error.
|
| +*/
|
| +static void jsonArrayFunc(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + int i;
|
| + JsonString jx;
|
| +
|
| + jsonInit(&jx, ctx);
|
| + jsonAppendChar(&jx, '[');
|
| + for(i=0; i<argc; i++){
|
| + jsonAppendSeparator(&jx);
|
| + jsonAppendValue(&jx, argv[i]);
|
| + }
|
| + jsonAppendChar(&jx, ']');
|
| + jsonResult(&jx);
|
| + sqlite3_result_subtype(ctx, JSON_SUBTYPE);
|
| +}
|
| +
|
| +
|
| +/*
|
| +** json_array_length(JSON)
|
| +** json_array_length(JSON, PATH)
|
| +**
|
| +** Return the number of elements in the top-level JSON array.
|
| +** Return 0 if the input is not a well-formed JSON array.
|
| +*/
|
| +static void jsonArrayLengthFunc(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + JsonParse x; /* The parse */
|
| + sqlite3_int64 n = 0;
|
| + u32 i;
|
| + JsonNode *pNode;
|
| +
|
| + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
|
| + assert( x.nNode );
|
| + if( argc==2 ){
|
| + const char *zPath = (const char*)sqlite3_value_text(argv[1]);
|
| + pNode = jsonLookup(&x, zPath, 0, ctx);
|
| + }else{
|
| + pNode = x.aNode;
|
| + }
|
| + if( pNode==0 ){
|
| + x.nErr = 1;
|
| + }else if( pNode->eType==JSON_ARRAY ){
|
| + assert( (pNode->jnFlags & JNODE_APPEND)==0 );
|
| + for(i=1; i<=pNode->n; n++){
|
| + i += jsonNodeSize(&pNode[i]);
|
| + }
|
| + }
|
| + if( x.nErr==0 ) sqlite3_result_int64(ctx, n);
|
| + jsonParseReset(&x);
|
| +}
|
| +
|
| +/*
|
| +** json_extract(JSON, PATH, ...)
|
| +**
|
| +** Return the element described by PATH. Return NULL if there is no
|
| +** PATH element. If there are multiple PATHs, then return a JSON array
|
| +** with the result from each path. Throw an error if the JSON or any PATH
|
| +** is malformed.
|
| +*/
|
| +static void jsonExtractFunc(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + JsonParse x; /* The parse */
|
| + JsonNode *pNode;
|
| + const char *zPath;
|
| + JsonString jx;
|
| + int i;
|
| +
|
| + if( argc<2 ) return;
|
| + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
|
| + jsonInit(&jx, ctx);
|
| + jsonAppendChar(&jx, '[');
|
| + for(i=1; i<argc; i++){
|
| + zPath = (const char*)sqlite3_value_text(argv[i]);
|
| + pNode = jsonLookup(&x, zPath, 0, ctx);
|
| + if( x.nErr ) break;
|
| + if( argc>2 ){
|
| + jsonAppendSeparator(&jx);
|
| + if( pNode ){
|
| + jsonRenderNode(pNode, &jx, 0);
|
| + }else{
|
| + jsonAppendRaw(&jx, "null", 4);
|
| + }
|
| + }else if( pNode ){
|
| + jsonReturn(pNode, ctx, 0);
|
| + }
|
| + }
|
| + if( argc>2 && i==argc ){
|
| + jsonAppendChar(&jx, ']');
|
| + jsonResult(&jx);
|
| + sqlite3_result_subtype(ctx, JSON_SUBTYPE);
|
| + }
|
| + jsonReset(&jx);
|
| + jsonParseReset(&x);
|
| +}
|
| +
|
| +/*
|
| +** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON
|
| +** object that contains all name/value given in arguments. Or if any name
|
| +** is not a string or if any value is a BLOB, throw an error.
|
| +*/
|
| +static void jsonObjectFunc(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + int i;
|
| + JsonString jx;
|
| + const char *z;
|
| + u32 n;
|
| +
|
| + if( argc&1 ){
|
| + sqlite3_result_error(ctx, "json_object() requires an even number "
|
| + "of arguments", -1);
|
| + return;
|
| + }
|
| + jsonInit(&jx, ctx);
|
| + jsonAppendChar(&jx, '{');
|
| + for(i=0; i<argc; i+=2){
|
| + if( sqlite3_value_type(argv[i])!=SQLITE_TEXT ){
|
| + sqlite3_result_error(ctx, "json_object() labels must be TEXT", -1);
|
| + jsonReset(&jx);
|
| + return;
|
| + }
|
| + jsonAppendSeparator(&jx);
|
| + z = (const char*)sqlite3_value_text(argv[i]);
|
| + n = (u32)sqlite3_value_bytes(argv[i]);
|
| + jsonAppendString(&jx, z, n);
|
| + jsonAppendChar(&jx, ':');
|
| + jsonAppendValue(&jx, argv[i+1]);
|
| + }
|
| + jsonAppendChar(&jx, '}');
|
| + jsonResult(&jx);
|
| + sqlite3_result_subtype(ctx, JSON_SUBTYPE);
|
| +}
|
| +
|
| +
|
| +/*
|
| +** json_remove(JSON, PATH, ...)
|
| +**
|
| +** Remove the named elements from JSON and return the result. malformed
|
| +** JSON or PATH arguments result in an error.
|
| +*/
|
| +static void jsonRemoveFunc(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + JsonParse x; /* The parse */
|
| + JsonNode *pNode;
|
| + const char *zPath;
|
| + u32 i;
|
| +
|
| + if( argc<1 ) return;
|
| + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
|
| + assert( x.nNode );
|
| + for(i=1; i<(u32)argc; i++){
|
| + zPath = (const char*)sqlite3_value_text(argv[i]);
|
| + if( zPath==0 ) goto remove_done;
|
| + pNode = jsonLookup(&x, zPath, 0, ctx);
|
| + if( x.nErr ) goto remove_done;
|
| + if( pNode ) pNode->jnFlags |= JNODE_REMOVE;
|
| + }
|
| + if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){
|
| + jsonReturnJson(x.aNode, ctx, 0);
|
| + }
|
| +remove_done:
|
| + jsonParseReset(&x);
|
| +}
|
| +
|
| +/*
|
| +** json_replace(JSON, PATH, VALUE, ...)
|
| +**
|
| +** Replace the value at PATH with VALUE. If PATH does not already exist,
|
| +** this routine is a no-op. If JSON or PATH is malformed, throw an error.
|
| +*/
|
| +static void jsonReplaceFunc(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + JsonParse x; /* The parse */
|
| + JsonNode *pNode;
|
| + const char *zPath;
|
| + u32 i;
|
| +
|
| + if( argc<1 ) return;
|
| + if( (argc&1)==0 ) {
|
| + jsonWrongNumArgs(ctx, "replace");
|
| + return;
|
| + }
|
| + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
|
| + assert( x.nNode );
|
| + for(i=1; i<(u32)argc; i+=2){
|
| + zPath = (const char*)sqlite3_value_text(argv[i]);
|
| + pNode = jsonLookup(&x, zPath, 0, ctx);
|
| + if( x.nErr ) goto replace_err;
|
| + if( pNode ){
|
| + pNode->jnFlags |= (u8)JNODE_REPLACE;
|
| + pNode->iVal = (u8)(i+1);
|
| + }
|
| + }
|
| + if( x.aNode[0].jnFlags & JNODE_REPLACE ){
|
| + sqlite3_result_value(ctx, argv[x.aNode[0].iVal]);
|
| + }else{
|
| + jsonReturnJson(x.aNode, ctx, argv);
|
| + }
|
| +replace_err:
|
| + jsonParseReset(&x);
|
| +}
|
| +
|
| +/*
|
| +** json_set(JSON, PATH, VALUE, ...)
|
| +**
|
| +** Set the value at PATH to VALUE. Create the PATH if it does not already
|
| +** exist. Overwrite existing values that do exist.
|
| +** If JSON or PATH is malformed, throw an error.
|
| +**
|
| +** json_insert(JSON, PATH, VALUE, ...)
|
| +**
|
| +** Create PATH and initialize it to VALUE. If PATH already exists, this
|
| +** routine is a no-op. If JSON or PATH is malformed, throw an error.
|
| +*/
|
| +static void jsonSetFunc(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + JsonParse x; /* The parse */
|
| + JsonNode *pNode;
|
| + const char *zPath;
|
| + u32 i;
|
| + int bApnd;
|
| + int bIsSet = *(int*)sqlite3_user_data(ctx);
|
| +
|
| + if( argc<1 ) return;
|
| + if( (argc&1)==0 ) {
|
| + jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert");
|
| + return;
|
| + }
|
| + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
|
| + assert( x.nNode );
|
| + for(i=1; i<(u32)argc; i+=2){
|
| + zPath = (const char*)sqlite3_value_text(argv[i]);
|
| + bApnd = 0;
|
| + pNode = jsonLookup(&x, zPath, &bApnd, ctx);
|
| + if( x.oom ){
|
| + sqlite3_result_error_nomem(ctx);
|
| + goto jsonSetDone;
|
| + }else if( x.nErr ){
|
| + goto jsonSetDone;
|
| + }else if( pNode && (bApnd || bIsSet) ){
|
| + pNode->jnFlags |= (u8)JNODE_REPLACE;
|
| + pNode->iVal = (u8)(i+1);
|
| + }
|
| + }
|
| + if( x.aNode[0].jnFlags & JNODE_REPLACE ){
|
| + sqlite3_result_value(ctx, argv[x.aNode[0].iVal]);
|
| + }else{
|
| + jsonReturnJson(x.aNode, ctx, argv);
|
| + }
|
| +jsonSetDone:
|
| + jsonParseReset(&x);
|
| +}
|
| +
|
| +/*
|
| +** json_type(JSON)
|
| +** json_type(JSON, PATH)
|
| +**
|
| +** Return the top-level "type" of a JSON string. Throw an error if
|
| +** either the JSON or PATH inputs are not well-formed.
|
| +*/
|
| +static void jsonTypeFunc(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + JsonParse x; /* The parse */
|
| + const char *zPath;
|
| + JsonNode *pNode;
|
| +
|
| + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
|
| + assert( x.nNode );
|
| + if( argc==2 ){
|
| + zPath = (const char*)sqlite3_value_text(argv[1]);
|
| + pNode = jsonLookup(&x, zPath, 0, ctx);
|
| + }else{
|
| + pNode = x.aNode;
|
| + }
|
| + if( pNode ){
|
| + sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC);
|
| + }
|
| + jsonParseReset(&x);
|
| +}
|
| +
|
| +/*
|
| +** json_valid(JSON)
|
| +**
|
| +** Return 1 if JSON is a well-formed JSON string according to RFC-7159.
|
| +** Return 0 otherwise.
|
| +*/
|
| +static void jsonValidFunc(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + JsonParse x; /* The parse */
|
| + int rc = 0;
|
| +
|
| + UNUSED_PARAM(argc);
|
| + if( jsonParse(&x, 0, (const char*)sqlite3_value_text(argv[0]))==0 ){
|
| + rc = 1;
|
| + }
|
| + jsonParseReset(&x);
|
| + sqlite3_result_int(ctx, rc);
|
| +}
|
| +
|
| +
|
| +/****************************************************************************
|
| +** Aggregate SQL function implementations
|
| +****************************************************************************/
|
| +/*
|
| +** json_group_array(VALUE)
|
| +**
|
| +** Return a JSON array composed of all values in the aggregate.
|
| +*/
|
| +static void jsonArrayStep(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + JsonString *pStr;
|
| + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr));
|
| + if( pStr ){
|
| + if( pStr->zBuf==0 ){
|
| + jsonInit(pStr, ctx);
|
| + jsonAppendChar(pStr, '[');
|
| + }else{
|
| + jsonAppendChar(pStr, ',');
|
| + pStr->pCtx = ctx;
|
| + }
|
| + jsonAppendValue(pStr, argv[0]);
|
| + }
|
| +}
|
| +static void jsonArrayFinal(sqlite3_context *ctx){
|
| + JsonString *pStr;
|
| + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
|
| + if( pStr ){
|
| + pStr->pCtx = ctx;
|
| + jsonAppendChar(pStr, ']');
|
| + if( pStr->bErr ){
|
| + sqlite3_result_error_nomem(ctx);
|
| + assert( pStr->bStatic );
|
| + }else{
|
| + sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
|
| + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
|
| + pStr->bStatic = 1;
|
| + }
|
| + }else{
|
| + sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC);
|
| + }
|
| + sqlite3_result_subtype(ctx, JSON_SUBTYPE);
|
| +}
|
| +
|
| +/*
|
| +** json_group_obj(NAME,VALUE)
|
| +**
|
| +** Return a JSON object composed of all names and values in the aggregate.
|
| +*/
|
| +static void jsonObjectStep(
|
| + sqlite3_context *ctx,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + JsonString *pStr;
|
| + const char *z;
|
| + u32 n;
|
| + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr));
|
| + if( pStr ){
|
| + if( pStr->zBuf==0 ){
|
| + jsonInit(pStr, ctx);
|
| + jsonAppendChar(pStr, '{');
|
| + }else{
|
| + jsonAppendChar(pStr, ',');
|
| + pStr->pCtx = ctx;
|
| + }
|
| + z = (const char*)sqlite3_value_text(argv[0]);
|
| + n = (u32)sqlite3_value_bytes(argv[0]);
|
| + jsonAppendString(pStr, z, n);
|
| + jsonAppendChar(pStr, ':');
|
| + jsonAppendValue(pStr, argv[1]);
|
| + }
|
| +}
|
| +static void jsonObjectFinal(sqlite3_context *ctx){
|
| + JsonString *pStr;
|
| + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
|
| + if( pStr ){
|
| + jsonAppendChar(pStr, '}');
|
| + if( pStr->bErr ){
|
| + sqlite3_result_error_nomem(ctx);
|
| + assert( pStr->bStatic );
|
| + }else{
|
| + sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
|
| + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
|
| + pStr->bStatic = 1;
|
| + }
|
| + }else{
|
| + sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC);
|
| + }
|
| + sqlite3_result_subtype(ctx, JSON_SUBTYPE);
|
| +}
|
| +
|
| +
|
| +#ifndef SQLITE_OMIT_VIRTUALTABLE
|
| +/****************************************************************************
|
| +** The json_each virtual table
|
| +****************************************************************************/
|
| +typedef struct JsonEachCursor JsonEachCursor;
|
| +struct JsonEachCursor {
|
| + sqlite3_vtab_cursor base; /* Base class - must be first */
|
| + u32 iRowid; /* The rowid */
|
| + u32 iBegin; /* The first node of the scan */
|
| + u32 i; /* Index in sParse.aNode[] of current row */
|
| + u32 iEnd; /* EOF when i equals or exceeds this value */
|
| + u8 eType; /* Type of top-level element */
|
| + u8 bRecursive; /* True for json_tree(). False for json_each() */
|
| + char *zJson; /* Input JSON */
|
| + char *zRoot; /* Path by which to filter zJson */
|
| + JsonParse sParse; /* Parse of the input JSON */
|
| +};
|
| +
|
| +/* Constructor for the json_each virtual table */
|
| +static int jsonEachConnect(
|
| + sqlite3 *db,
|
| + void *pAux,
|
| + int argc, const char *const*argv,
|
| + sqlite3_vtab **ppVtab,
|
| + char **pzErr
|
| +){
|
| + sqlite3_vtab *pNew;
|
| + int rc;
|
| +
|
| +/* Column numbers */
|
| +#define JEACH_KEY 0
|
| +#define JEACH_VALUE 1
|
| +#define JEACH_TYPE 2
|
| +#define JEACH_ATOM 3
|
| +#define JEACH_ID 4
|
| +#define JEACH_PARENT 5
|
| +#define JEACH_FULLKEY 6
|
| +#define JEACH_PATH 7
|
| +#define JEACH_JSON 8
|
| +#define JEACH_ROOT 9
|
| +
|
| + UNUSED_PARAM(pzErr);
|
| + UNUSED_PARAM(argv);
|
| + UNUSED_PARAM(argc);
|
| + UNUSED_PARAM(pAux);
|
| + rc = sqlite3_declare_vtab(db,
|
| + "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path,"
|
| + "json HIDDEN,root HIDDEN)");
|
| + if( rc==SQLITE_OK ){
|
| + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) );
|
| + if( pNew==0 ) return SQLITE_NOMEM;
|
| + memset(pNew, 0, sizeof(*pNew));
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/* destructor for json_each virtual table */
|
| +static int jsonEachDisconnect(sqlite3_vtab *pVtab){
|
| + sqlite3_free(pVtab);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* constructor for a JsonEachCursor object for json_each(). */
|
| +static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
|
| + JsonEachCursor *pCur;
|
| +
|
| + UNUSED_PARAM(p);
|
| + pCur = sqlite3_malloc( sizeof(*pCur) );
|
| + if( pCur==0 ) return SQLITE_NOMEM;
|
| + memset(pCur, 0, sizeof(*pCur));
|
| + *ppCursor = &pCur->base;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* constructor for a JsonEachCursor object for json_tree(). */
|
| +static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
|
| + int rc = jsonEachOpenEach(p, ppCursor);
|
| + if( rc==SQLITE_OK ){
|
| + JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor;
|
| + pCur->bRecursive = 1;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/* Reset a JsonEachCursor back to its original state. Free any memory
|
| +** held. */
|
| +static void jsonEachCursorReset(JsonEachCursor *p){
|
| + sqlite3_free(p->zJson);
|
| + sqlite3_free(p->zRoot);
|
| + jsonParseReset(&p->sParse);
|
| + p->iRowid = 0;
|
| + p->i = 0;
|
| + p->iEnd = 0;
|
| + p->eType = 0;
|
| + p->zJson = 0;
|
| + p->zRoot = 0;
|
| +}
|
| +
|
| +/* Destructor for a jsonEachCursor object */
|
| +static int jsonEachClose(sqlite3_vtab_cursor *cur){
|
| + JsonEachCursor *p = (JsonEachCursor*)cur;
|
| + jsonEachCursorReset(p);
|
| + sqlite3_free(cur);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Return TRUE if the jsonEachCursor object has been advanced off the end
|
| +** of the JSON object */
|
| +static int jsonEachEof(sqlite3_vtab_cursor *cur){
|
| + JsonEachCursor *p = (JsonEachCursor*)cur;
|
| + return p->i >= p->iEnd;
|
| +}
|
| +
|
| +/* Advance the cursor to the next element for json_tree() */
|
| +static int jsonEachNext(sqlite3_vtab_cursor *cur){
|
| + JsonEachCursor *p = (JsonEachCursor*)cur;
|
| + if( p->bRecursive ){
|
| + if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++;
|
| + p->i++;
|
| + p->iRowid++;
|
| + if( p->i<p->iEnd ){
|
| + u32 iUp = p->sParse.aUp[p->i];
|
| + JsonNode *pUp = &p->sParse.aNode[iUp];
|
| + p->eType = pUp->eType;
|
| + if( pUp->eType==JSON_ARRAY ){
|
| + if( iUp==p->i-1 ){
|
| + pUp->u.iKey = 0;
|
| + }else{
|
| + pUp->u.iKey++;
|
| + }
|
| + }
|
| + }
|
| + }else{
|
| + switch( p->eType ){
|
| + case JSON_ARRAY: {
|
| + p->i += jsonNodeSize(&p->sParse.aNode[p->i]);
|
| + p->iRowid++;
|
| + break;
|
| + }
|
| + case JSON_OBJECT: {
|
| + p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]);
|
| + p->iRowid++;
|
| + break;
|
| + }
|
| + default: {
|
| + p->i = p->iEnd;
|
| + break;
|
| + }
|
| + }
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Append the name of the path for element i to pStr
|
| +*/
|
| +static void jsonEachComputePath(
|
| + JsonEachCursor *p, /* The cursor */
|
| + JsonString *pStr, /* Write the path here */
|
| + u32 i /* Path to this element */
|
| +){
|
| + JsonNode *pNode, *pUp;
|
| + u32 iUp;
|
| + if( i==0 ){
|
| + jsonAppendChar(pStr, '$');
|
| + return;
|
| + }
|
| + iUp = p->sParse.aUp[i];
|
| + jsonEachComputePath(p, pStr, iUp);
|
| + pNode = &p->sParse.aNode[i];
|
| + pUp = &p->sParse.aNode[iUp];
|
| + if( pUp->eType==JSON_ARRAY ){
|
| + jsonPrintf(30, pStr, "[%d]", pUp->u.iKey);
|
| + }else{
|
| + assert( pUp->eType==JSON_OBJECT );
|
| + if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--;
|
| + assert( pNode->eType==JSON_STRING );
|
| + assert( pNode->jnFlags & JNODE_LABEL );
|
| + jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1);
|
| + }
|
| +}
|
| +
|
| +/* Return the value of a column */
|
| +static int jsonEachColumn(
|
| + sqlite3_vtab_cursor *cur, /* The cursor */
|
| + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
|
| + int i /* Which column to return */
|
| +){
|
| + JsonEachCursor *p = (JsonEachCursor*)cur;
|
| + JsonNode *pThis = &p->sParse.aNode[p->i];
|
| + switch( i ){
|
| + case JEACH_KEY: {
|
| + if( p->i==0 ) break;
|
| + if( p->eType==JSON_OBJECT ){
|
| + jsonReturn(pThis, ctx, 0);
|
| + }else if( p->eType==JSON_ARRAY ){
|
| + u32 iKey;
|
| + if( p->bRecursive ){
|
| + if( p->iRowid==0 ) break;
|
| + iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey;
|
| + }else{
|
| + iKey = p->iRowid;
|
| + }
|
| + sqlite3_result_int64(ctx, (sqlite3_int64)iKey);
|
| + }
|
| + break;
|
| + }
|
| + case JEACH_VALUE: {
|
| + if( pThis->jnFlags & JNODE_LABEL ) pThis++;
|
| + jsonReturn(pThis, ctx, 0);
|
| + break;
|
| + }
|
| + case JEACH_TYPE: {
|
| + if( pThis->jnFlags & JNODE_LABEL ) pThis++;
|
| + sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC);
|
| + break;
|
| + }
|
| + case JEACH_ATOM: {
|
| + if( pThis->jnFlags & JNODE_LABEL ) pThis++;
|
| + if( pThis->eType>=JSON_ARRAY ) break;
|
| + jsonReturn(pThis, ctx, 0);
|
| + break;
|
| + }
|
| + case JEACH_ID: {
|
| + sqlite3_result_int64(ctx,
|
| + (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0));
|
| + break;
|
| + }
|
| + case JEACH_PARENT: {
|
| + if( p->i>p->iBegin && p->bRecursive ){
|
| + sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]);
|
| + }
|
| + break;
|
| + }
|
| + case JEACH_FULLKEY: {
|
| + JsonString x;
|
| + jsonInit(&x, ctx);
|
| + if( p->bRecursive ){
|
| + jsonEachComputePath(p, &x, p->i);
|
| + }else{
|
| + if( p->zRoot ){
|
| + jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot));
|
| + }else{
|
| + jsonAppendChar(&x, '$');
|
| + }
|
| + if( p->eType==JSON_ARRAY ){
|
| + jsonPrintf(30, &x, "[%d]", p->iRowid);
|
| + }else{
|
| + jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1);
|
| + }
|
| + }
|
| + jsonResult(&x);
|
| + break;
|
| + }
|
| + case JEACH_PATH: {
|
| + if( p->bRecursive ){
|
| + JsonString x;
|
| + jsonInit(&x, ctx);
|
| + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]);
|
| + jsonResult(&x);
|
| + break;
|
| + }
|
| + /* For json_each() path and root are the same so fall through
|
| + ** into the root case */
|
| + }
|
| + case JEACH_ROOT: {
|
| + const char *zRoot = p->zRoot;
|
| + if( zRoot==0 ) zRoot = "$";
|
| + sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC);
|
| + break;
|
| + }
|
| + case JEACH_JSON: {
|
| + assert( i==JEACH_JSON );
|
| + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC);
|
| + break;
|
| + }
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Return the current rowid value */
|
| +static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
|
| + JsonEachCursor *p = (JsonEachCursor*)cur;
|
| + *pRowid = p->iRowid;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* The query strategy is to look for an equality constraint on the json
|
| +** column. Without such a constraint, the table cannot operate. idxNum is
|
| +** 1 if the constraint is found, 3 if the constraint and zRoot are found,
|
| +** and 0 otherwise.
|
| +*/
|
| +static int jsonEachBestIndex(
|
| + sqlite3_vtab *tab,
|
| + sqlite3_index_info *pIdxInfo
|
| +){
|
| + int i;
|
| + int jsonIdx = -1;
|
| + int rootIdx = -1;
|
| + const struct sqlite3_index_constraint *pConstraint;
|
| +
|
| + UNUSED_PARAM(tab);
|
| + pConstraint = pIdxInfo->aConstraint;
|
| + for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
|
| + if( pConstraint->usable==0 ) continue;
|
| + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
|
| + switch( pConstraint->iColumn ){
|
| + case JEACH_JSON: jsonIdx = i; break;
|
| + case JEACH_ROOT: rootIdx = i; break;
|
| + default: /* no-op */ break;
|
| + }
|
| + }
|
| + if( jsonIdx<0 ){
|
| + pIdxInfo->idxNum = 0;
|
| + pIdxInfo->estimatedCost = 1e99;
|
| + }else{
|
| + pIdxInfo->estimatedCost = 1.0;
|
| + pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1;
|
| + pIdxInfo->aConstraintUsage[jsonIdx].omit = 1;
|
| + if( rootIdx<0 ){
|
| + pIdxInfo->idxNum = 1;
|
| + }else{
|
| + pIdxInfo->aConstraintUsage[rootIdx].argvIndex = 2;
|
| + pIdxInfo->aConstraintUsage[rootIdx].omit = 1;
|
| + pIdxInfo->idxNum = 3;
|
| + }
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* Start a search on a new JSON string */
|
| +static int jsonEachFilter(
|
| + sqlite3_vtab_cursor *cur,
|
| + int idxNum, const char *idxStr,
|
| + int argc, sqlite3_value **argv
|
| +){
|
| + JsonEachCursor *p = (JsonEachCursor*)cur;
|
| + const char *z;
|
| + const char *zRoot = 0;
|
| + sqlite3_int64 n;
|
| +
|
| + UNUSED_PARAM(idxStr);
|
| + UNUSED_PARAM(argc);
|
| + jsonEachCursorReset(p);
|
| + if( idxNum==0 ) return SQLITE_OK;
|
| + z = (const char*)sqlite3_value_text(argv[0]);
|
| + if( z==0 ) return SQLITE_OK;
|
| + n = sqlite3_value_bytes(argv[0]);
|
| + p->zJson = sqlite3_malloc64( n+1 );
|
| + if( p->zJson==0 ) return SQLITE_NOMEM;
|
| + memcpy(p->zJson, z, (size_t)n+1);
|
| + if( jsonParse(&p->sParse, 0, p->zJson) ){
|
| + int rc = SQLITE_NOMEM;
|
| + if( p->sParse.oom==0 ){
|
| + sqlite3_free(cur->pVtab->zErrMsg);
|
| + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON");
|
| + if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR;
|
| + }
|
| + jsonEachCursorReset(p);
|
| + return rc;
|
| + }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){
|
| + jsonEachCursorReset(p);
|
| + return SQLITE_NOMEM;
|
| + }else{
|
| + JsonNode *pNode = 0;
|
| + if( idxNum==3 ){
|
| + const char *zErr = 0;
|
| + zRoot = (const char*)sqlite3_value_text(argv[1]);
|
| + if( zRoot==0 ) return SQLITE_OK;
|
| + n = sqlite3_value_bytes(argv[1]);
|
| + p->zRoot = sqlite3_malloc64( n+1 );
|
| + if( p->zRoot==0 ) return SQLITE_NOMEM;
|
| + memcpy(p->zRoot, zRoot, (size_t)n+1);
|
| + if( zRoot[0]!='$' ){
|
| + zErr = zRoot;
|
| + }else{
|
| + pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr);
|
| + }
|
| + if( zErr ){
|
| + sqlite3_free(cur->pVtab->zErrMsg);
|
| + cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr);
|
| + jsonEachCursorReset(p);
|
| + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM;
|
| + }else if( pNode==0 ){
|
| + return SQLITE_OK;
|
| + }
|
| + }else{
|
| + pNode = p->sParse.aNode;
|
| + }
|
| + p->iBegin = p->i = (int)(pNode - p->sParse.aNode);
|
| + p->eType = pNode->eType;
|
| + if( p->eType>=JSON_ARRAY ){
|
| + pNode->u.iKey = 0;
|
| + p->iEnd = p->i + pNode->n + 1;
|
| + if( p->bRecursive ){
|
| + p->eType = p->sParse.aNode[p->sParse.aUp[p->i]].eType;
|
| + if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){
|
| + p->i--;
|
| + }
|
| + }else{
|
| + p->i++;
|
| + }
|
| + }else{
|
| + p->iEnd = p->i+1;
|
| + }
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/* The methods of the json_each virtual table */
|
| +static sqlite3_module jsonEachModule = {
|
| + 0, /* iVersion */
|
| + 0, /* xCreate */
|
| + jsonEachConnect, /* xConnect */
|
| + jsonEachBestIndex, /* xBestIndex */
|
| + jsonEachDisconnect, /* xDisconnect */
|
| + 0, /* xDestroy */
|
| + jsonEachOpenEach, /* xOpen - open a cursor */
|
| + jsonEachClose, /* xClose - close a cursor */
|
| + jsonEachFilter, /* xFilter - configure scan constraints */
|
| + jsonEachNext, /* xNext - advance a cursor */
|
| + jsonEachEof, /* xEof - check for end of scan */
|
| + jsonEachColumn, /* xColumn - read data */
|
| + jsonEachRowid, /* xRowid - read data */
|
| + 0, /* xUpdate */
|
| + 0, /* xBegin */
|
| + 0, /* xSync */
|
| + 0, /* xCommit */
|
| + 0, /* xRollback */
|
| + 0, /* xFindMethod */
|
| + 0, /* xRename */
|
| + 0, /* xSavepoint */
|
| + 0, /* xRelease */
|
| + 0 /* xRollbackTo */
|
| +};
|
| +
|
| +/* The methods of the json_tree virtual table. */
|
| +static sqlite3_module jsonTreeModule = {
|
| + 0, /* iVersion */
|
| + 0, /* xCreate */
|
| + jsonEachConnect, /* xConnect */
|
| + jsonEachBestIndex, /* xBestIndex */
|
| + jsonEachDisconnect, /* xDisconnect */
|
| + 0, /* xDestroy */
|
| + jsonEachOpenTree, /* xOpen - open a cursor */
|
| + jsonEachClose, /* xClose - close a cursor */
|
| + jsonEachFilter, /* xFilter - configure scan constraints */
|
| + jsonEachNext, /* xNext - advance a cursor */
|
| + jsonEachEof, /* xEof - check for end of scan */
|
| + jsonEachColumn, /* xColumn - read data */
|
| + jsonEachRowid, /* xRowid - read data */
|
| + 0, /* xUpdate */
|
| + 0, /* xBegin */
|
| + 0, /* xSync */
|
| + 0, /* xCommit */
|
| + 0, /* xRollback */
|
| + 0, /* xFindMethod */
|
| + 0, /* xRename */
|
| + 0, /* xSavepoint */
|
| + 0, /* xRelease */
|
| + 0 /* xRollbackTo */
|
| +};
|
| +#endif /* SQLITE_OMIT_VIRTUALTABLE */
|
| +
|
| +/****************************************************************************
|
| +** The following routines are the only publically visible identifiers in this
|
| +** file. Call the following routines in order to register the various SQL
|
| +** functions and the virtual table implemented by this file.
|
| +****************************************************************************/
|
| +
|
| +SQLITE_PRIVATE int sqlite3Json1Init(sqlite3 *db){
|
| + int rc = SQLITE_OK;
|
| + unsigned int i;
|
| + static const struct {
|
| + const char *zName;
|
| + int nArg;
|
| + int flag;
|
| + void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
|
| + } aFunc[] = {
|
| + { "json", 1, 0, jsonRemoveFunc },
|
| + { "json_array", -1, 0, jsonArrayFunc },
|
| + { "json_array_length", 1, 0, jsonArrayLengthFunc },
|
| + { "json_array_length", 2, 0, jsonArrayLengthFunc },
|
| + { "json_extract", -1, 0, jsonExtractFunc },
|
| + { "json_insert", -1, 0, jsonSetFunc },
|
| + { "json_object", -1, 0, jsonObjectFunc },
|
| + { "json_remove", -1, 0, jsonRemoveFunc },
|
| + { "json_replace", -1, 0, jsonReplaceFunc },
|
| + { "json_set", -1, 1, jsonSetFunc },
|
| + { "json_type", 1, 0, jsonTypeFunc },
|
| + { "json_type", 2, 0, jsonTypeFunc },
|
| + { "json_valid", 1, 0, jsonValidFunc },
|
| +
|
| +#if SQLITE_DEBUG
|
| + /* DEBUG and TESTING functions */
|
| + { "json_parse", 1, 0, jsonParseFunc },
|
| + { "json_test1", 1, 0, jsonTest1Func },
|
| +#endif
|
| + };
|
| + static const struct {
|
| + const char *zName;
|
| + int nArg;
|
| + void (*xStep)(sqlite3_context*,int,sqlite3_value**);
|
| + void (*xFinal)(sqlite3_context*);
|
| + } aAgg[] = {
|
| + { "json_group_array", 1, jsonArrayStep, jsonArrayFinal },
|
| + { "json_group_object", 2, jsonObjectStep, jsonObjectFinal },
|
| + };
|
| +#ifndef SQLITE_OMIT_VIRTUALTABLE
|
| + static const struct {
|
| + const char *zName;
|
| + sqlite3_module *pModule;
|
| + } aMod[] = {
|
| + { "json_each", &jsonEachModule },
|
| + { "json_tree", &jsonTreeModule },
|
| + };
|
| +#endif
|
| + for(i=0; i<sizeof(aFunc)/sizeof(aFunc[0]) && rc==SQLITE_OK; i++){
|
| + rc = sqlite3_create_function(db, aFunc[i].zName, aFunc[i].nArg,
|
| + SQLITE_UTF8 | SQLITE_DETERMINISTIC,
|
| + (void*)&aFunc[i].flag,
|
| + aFunc[i].xFunc, 0, 0);
|
| + }
|
| + for(i=0; i<sizeof(aAgg)/sizeof(aAgg[0]) && rc==SQLITE_OK; i++){
|
| + rc = sqlite3_create_function(db, aAgg[i].zName, aAgg[i].nArg,
|
| + SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0,
|
| + 0, aAgg[i].xStep, aAgg[i].xFinal);
|
| + }
|
| +#ifndef SQLITE_OMIT_VIRTUALTABLE
|
| + for(i=0; i<sizeof(aMod)/sizeof(aMod[0]) && rc==SQLITE_OK; i++){
|
| + rc = sqlite3_create_module(db, aMod[i].zName, aMod[i].pModule, 0);
|
| + }
|
| +#endif
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +#ifndef SQLITE_CORE
|
| +#ifdef _WIN32
|
| +__declspec(dllexport)
|
| +#endif
|
| +SQLITE_API int SQLITE_STDCALL sqlite3_json_init(
|
| + sqlite3 *db,
|
| + char **pzErrMsg,
|
| + const sqlite3_api_routines *pApi
|
| +){
|
| + SQLITE_EXTENSION_INIT2(pApi);
|
| + (void)pzErrMsg; /* Unused parameter */
|
| + return sqlite3Json1Init(db);
|
| +}
|
| +#endif
|
| +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) */
|
| +
|
| +/************** End of json1.c ***********************************************/
|
| +/************** Begin file fts5.c ********************************************/
|
| +
|
| +
|
| +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5)
|
| +
|
| +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
|
| +# define NDEBUG 1
|
| +#endif
|
| +#if defined(NDEBUG) && defined(SQLITE_DEBUG)
|
| +# undef NDEBUG
|
| +#endif
|
| +
|
| +/*
|
| +** 2014 May 31
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +** Interfaces to extend FTS5. Using the interfaces defined in this file,
|
| +** FTS5 may be extended with:
|
| +**
|
| +** * custom tokenizers, and
|
| +** * custom auxiliary functions.
|
| +*/
|
| +
|
| +
|
| +#ifndef _FTS5_H
|
| +#define _FTS5_H
|
| +
|
| +/* #include "sqlite3.h" */
|
| +
|
| +#if 0
|
| +extern "C" {
|
| +#endif
|
| +
|
| +/*************************************************************************
|
| +** CUSTOM AUXILIARY FUNCTIONS
|
| +**
|
| +** Virtual table implementations may overload SQL functions by implementing
|
| +** the sqlite3_module.xFindFunction() method.
|
| +*/
|
| +
|
| +typedef struct Fts5ExtensionApi Fts5ExtensionApi;
|
| +typedef struct Fts5Context Fts5Context;
|
| +typedef struct Fts5PhraseIter Fts5PhraseIter;
|
| +
|
| +typedef void (*fts5_extension_function)(
|
| + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
|
| + Fts5Context *pFts, /* First arg to pass to pApi functions */
|
| + sqlite3_context *pCtx, /* Context for returning result/error */
|
| + int nVal, /* Number of values in apVal[] array */
|
| + sqlite3_value **apVal /* Array of trailing arguments */
|
| +);
|
| +
|
| +struct Fts5PhraseIter {
|
| + const unsigned char *a;
|
| + const unsigned char *b;
|
| +};
|
| +
|
| +/*
|
| +** EXTENSION API FUNCTIONS
|
| +**
|
| +** xUserData(pFts):
|
| +** Return a copy of the context pointer the extension function was
|
| +** registered with.
|
| +**
|
| +** xColumnTotalSize(pFts, iCol, pnToken):
|
| +** If parameter iCol is less than zero, set output variable *pnToken
|
| +** to the total number of tokens in the FTS5 table. Or, if iCol is
|
| +** non-negative but less than the number of columns in the table, return
|
| +** the total number of tokens in column iCol, considering all rows in
|
| +** the FTS5 table.
|
| +**
|
| +** If parameter iCol is greater than or equal to the number of columns
|
| +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
|
| +** an OOM condition or IO error), an appropriate SQLite error code is
|
| +** returned.
|
| +**
|
| +** xColumnCount(pFts):
|
| +** Return the number of columns in the table.
|
| +**
|
| +** xColumnSize(pFts, iCol, pnToken):
|
| +** If parameter iCol is less than zero, set output variable *pnToken
|
| +** to the total number of tokens in the current row. Or, if iCol is
|
| +** non-negative but less than the number of columns in the table, set
|
| +** *pnToken to the number of tokens in column iCol of the current row.
|
| +**
|
| +** If parameter iCol is greater than or equal to the number of columns
|
| +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
|
| +** an OOM condition or IO error), an appropriate SQLite error code is
|
| +** returned.
|
| +**
|
| +** xColumnText:
|
| +** This function attempts to retrieve the text of column iCol of the
|
| +** current document. If successful, (*pz) is set to point to a buffer
|
| +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes
|
| +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
|
| +** if an error occurs, an SQLite error code is returned and the final values
|
| +** of (*pz) and (*pn) are undefined.
|
| +**
|
| +** xPhraseCount:
|
| +** Returns the number of phrases in the current query expression.
|
| +**
|
| +** xPhraseSize:
|
| +** Returns the number of tokens in phrase iPhrase of the query. Phrases
|
| +** are numbered starting from zero.
|
| +**
|
| +** xInstCount:
|
| +** Set *pnInst to the total number of occurrences of all phrases within
|
| +** the query within the current row. Return SQLITE_OK if successful, or
|
| +** an error code (i.e. SQLITE_NOMEM) if an error occurs.
|
| +**
|
| +** xInst:
|
| +** Query for the details of phrase match iIdx within the current row.
|
| +** Phrase matches are numbered starting from zero, so the iIdx argument
|
| +** should be greater than or equal to zero and smaller than the value
|
| +** output by xInstCount().
|
| +**
|
| +** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM)
|
| +** if an error occurs.
|
| +**
|
| +** xRowid:
|
| +** Returns the rowid of the current row.
|
| +**
|
| +** xTokenize:
|
| +** Tokenize text using the tokenizer belonging to the FTS5 table.
|
| +**
|
| +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
|
| +** This API function is used to query the FTS table for phrase iPhrase
|
| +** of the current query. Specifically, a query equivalent to:
|
| +**
|
| +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid
|
| +**
|
| +** with $p set to a phrase equivalent to the phrase iPhrase of the
|
| +** current query is executed. For each row visited, the callback function
|
| +** passed as the fourth argument is invoked. The context and API objects
|
| +** passed to the callback function may be used to access the properties of
|
| +** each matched row. Invoking Api.xUserData() returns a copy of the pointer
|
| +** passed as the third argument to pUserData.
|
| +**
|
| +** If the callback function returns any value other than SQLITE_OK, the
|
| +** query is abandoned and the xQueryPhrase function returns immediately.
|
| +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
|
| +** Otherwise, the error code is propagated upwards.
|
| +**
|
| +** If the query runs to completion without incident, SQLITE_OK is returned.
|
| +** Or, if some error occurs before the query completes or is aborted by
|
| +** the callback, an SQLite error code is returned.
|
| +**
|
| +**
|
| +** xSetAuxdata(pFts5, pAux, xDelete)
|
| +**
|
| +** Save the pointer passed as the second argument as the extension functions
|
| +** "auxiliary data". The pointer may then be retrieved by the current or any
|
| +** future invocation of the same fts5 extension function made as part of
|
| +** of the same MATCH query using the xGetAuxdata() API.
|
| +**
|
| +** Each extension function is allocated a single auxiliary data slot for
|
| +** each FTS query (MATCH expression). If the extension function is invoked
|
| +** more than once for a single FTS query, then all invocations share a
|
| +** single auxiliary data context.
|
| +**
|
| +** If there is already an auxiliary data pointer when this function is
|
| +** invoked, then it is replaced by the new pointer. If an xDelete callback
|
| +** was specified along with the original pointer, it is invoked at this
|
| +** point.
|
| +**
|
| +** The xDelete callback, if one is specified, is also invoked on the
|
| +** auxiliary data pointer after the FTS5 query has finished.
|
| +**
|
| +** If an error (e.g. an OOM condition) occurs within this function, an
|
| +** the auxiliary data is set to NULL and an error code returned. If the
|
| +** xDelete parameter was not NULL, it is invoked on the auxiliary data
|
| +** pointer before returning.
|
| +**
|
| +**
|
| +** xGetAuxdata(pFts5, bClear)
|
| +**
|
| +** Returns the current auxiliary data pointer for the fts5 extension
|
| +** function. See the xSetAuxdata() method for details.
|
| +**
|
| +** If the bClear argument is non-zero, then the auxiliary data is cleared
|
| +** (set to NULL) before this function returns. In this case the xDelete,
|
| +** if any, is not invoked.
|
| +**
|
| +**
|
| +** xRowCount(pFts5, pnRow)
|
| +**
|
| +** This function is used to retrieve the total number of rows in the table.
|
| +** In other words, the same value that would be returned by:
|
| +**
|
| +** SELECT count(*) FROM ftstable;
|
| +**
|
| +** xPhraseFirst()
|
| +** This function is used, along with type Fts5PhraseIter and the xPhraseNext
|
| +** method, to iterate through all instances of a single query phrase within
|
| +** the current row. This is the same information as is accessible via the
|
| +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient
|
| +** to use, this API may be faster under some circumstances. To iterate
|
| +** through instances of phrase iPhrase, use the following code:
|
| +**
|
| +** Fts5PhraseIter iter;
|
| +** int iCol, iOff;
|
| +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
|
| +** iOff>=0;
|
| +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
|
| +** ){
|
| +** // An instance of phrase iPhrase at offset iOff of column iCol
|
| +** }
|
| +**
|
| +** The Fts5PhraseIter structure is defined above. Applications should not
|
| +** modify this structure directly - it should only be used as shown above
|
| +** with the xPhraseFirst() and xPhraseNext() API methods.
|
| +**
|
| +** xPhraseNext()
|
| +** See xPhraseFirst above.
|
| +*/
|
| +struct Fts5ExtensionApi {
|
| + int iVersion; /* Currently always set to 1 */
|
| +
|
| + void *(*xUserData)(Fts5Context*);
|
| +
|
| + int (*xColumnCount)(Fts5Context*);
|
| + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
|
| + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);
|
| +
|
| + int (*xTokenize)(Fts5Context*,
|
| + const char *pText, int nText, /* Text to tokenize */
|
| + void *pCtx, /* Context passed to xToken() */
|
| + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
|
| + );
|
| +
|
| + int (*xPhraseCount)(Fts5Context*);
|
| + int (*xPhraseSize)(Fts5Context*, int iPhrase);
|
| +
|
| + int (*xInstCount)(Fts5Context*, int *pnInst);
|
| + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff);
|
| +
|
| + sqlite3_int64 (*xRowid)(Fts5Context*);
|
| + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
|
| + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
|
| +
|
| + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
|
| + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
|
| + );
|
| + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
|
| + void *(*xGetAuxdata)(Fts5Context*, int bClear);
|
| +
|
| + void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
|
| + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);
|
| +};
|
| +
|
| +/*
|
| +** CUSTOM AUXILIARY FUNCTIONS
|
| +*************************************************************************/
|
| +
|
| +/*************************************************************************
|
| +** CUSTOM TOKENIZERS
|
| +**
|
| +** Applications may also register custom tokenizer types. A tokenizer
|
| +** is registered by providing fts5 with a populated instance of the
|
| +** following structure. All structure methods must be defined, setting
|
| +** any member of the fts5_tokenizer struct to NULL leads to undefined
|
| +** behaviour. The structure methods are expected to function as follows:
|
| +**
|
| +** xCreate:
|
| +** This function is used to allocate and inititalize a tokenizer instance.
|
| +** A tokenizer instance is required to actually tokenize text.
|
| +**
|
| +** The first argument passed to this function is a copy of the (void*)
|
| +** pointer provided by the application when the fts5_tokenizer object
|
| +** was registered with FTS5 (the third argument to xCreateTokenizer()).
|
| +** The second and third arguments are an array of nul-terminated strings
|
| +** containing the tokenizer arguments, if any, specified following the
|
| +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used
|
| +** to create the FTS5 table.
|
| +**
|
| +** The final argument is an output variable. If successful, (*ppOut)
|
| +** should be set to point to the new tokenizer handle and SQLITE_OK
|
| +** returned. If an error occurs, some value other than SQLITE_OK should
|
| +** be returned. In this case, fts5 assumes that the final value of *ppOut
|
| +** is undefined.
|
| +**
|
| +** xDelete:
|
| +** This function is invoked to delete a tokenizer handle previously
|
| +** allocated using xCreate(). Fts5 guarantees that this function will
|
| +** be invoked exactly once for each successful call to xCreate().
|
| +**
|
| +** xTokenize:
|
| +** This function is expected to tokenize the nText byte string indicated
|
| +** by argument pText. pText may or may not be nul-terminated. The first
|
| +** argument passed to this function is a pointer to an Fts5Tokenizer object
|
| +** returned by an earlier call to xCreate().
|
| +**
|
| +** The second argument indicates the reason that FTS5 is requesting
|
| +** tokenization of the supplied text. This is always one of the following
|
| +** four values:
|
| +**
|
| +** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into
|
| +** or removed from the FTS table. The tokenizer is being invoked to
|
| +** determine the set of tokens to add to (or delete from) the
|
| +** FTS index.
|
| +**
|
| +** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed
|
| +** against the FTS index. The tokenizer is being called to tokenize
|
| +** a bareword or quoted string specified as part of the query.
|
| +**
|
| +** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as
|
| +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is
|
| +** followed by a "*" character, indicating that the last token
|
| +** returned by the tokenizer will be treated as a token prefix.
|
| +**
|
| +** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to
|
| +** satisfy an fts5_api.xTokenize() request made by an auxiliary
|
| +** function. Or an fts5_api.xColumnSize() request made by the same
|
| +** on a columnsize=0 database.
|
| +** </ul>
|
| +**
|
| +** For each token in the input string, the supplied callback xToken() must
|
| +** be invoked. The first argument to it should be a copy of the pointer
|
| +** passed as the second argument to xTokenize(). The third and fourth
|
| +** arguments are a pointer to a buffer containing the token text, and the
|
| +** size of the token in bytes. The 4th and 5th arguments are the byte offsets
|
| +** of the first byte of and first byte immediately following the text from
|
| +** which the token is derived within the input.
|
| +**
|
| +** The second argument passed to the xToken() callback ("tflags") should
|
| +** normally be set to 0. The exception is if the tokenizer supports
|
| +** synonyms. In this case see the discussion below for details.
|
| +**
|
| +** FTS5 assumes the xToken() callback is invoked for each token in the
|
| +** order that they occur within the input text.
|
| +**
|
| +** If an xToken() callback returns any value other than SQLITE_OK, then
|
| +** the tokenization should be abandoned and the xTokenize() method should
|
| +** immediately return a copy of the xToken() return value. Or, if the
|
| +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally,
|
| +** if an error occurs with the xTokenize() implementation itself, it
|
| +** may abandon the tokenization and return any error code other than
|
| +** SQLITE_OK or SQLITE_DONE.
|
| +**
|
| +** SYNONYM SUPPORT
|
| +**
|
| +** Custom tokenizers may also support synonyms. Consider a case in which a
|
| +** user wishes to query for a phrase such as "first place". Using the
|
| +** built-in tokenizers, the FTS5 query 'first + place' will match instances
|
| +** of "first place" within the document set, but not alternative forms
|
| +** such as "1st place". In some applications, it would be better to match
|
| +** all instances of "first place" or "1st place" regardless of which form
|
| +** the user specified in the MATCH query text.
|
| +**
|
| +** There are several ways to approach this in FTS5:
|
| +**
|
| +** <ol><li> By mapping all synonyms to a single token. In this case, the
|
| +** In the above example, this means that the tokenizer returns the
|
| +** same token for inputs "first" and "1st". Say that token is in
|
| +** fact "first", so that when the user inserts the document "I won
|
| +** 1st place" entries are added to the index for tokens "i", "won",
|
| +** "first" and "place". If the user then queries for '1st + place',
|
| +** the tokenizer substitutes "first" for "1st" and the query works
|
| +** as expected.
|
| +**
|
| +** <li> By adding multiple synonyms for a single term to the FTS index.
|
| +** In this case, when tokenizing query text, the tokenizer may
|
| +** provide multiple synonyms for a single term within the document.
|
| +** FTS5 then queries the index for each synonym individually. For
|
| +** example, faced with the query:
|
| +**
|
| +** <codeblock>
|
| +** ... MATCH 'first place'</codeblock>
|
| +**
|
| +** the tokenizer offers both "1st" and "first" as synonyms for the
|
| +** first token in the MATCH query and FTS5 effectively runs a query
|
| +** similar to:
|
| +**
|
| +** <codeblock>
|
| +** ... MATCH '(first OR 1st) place'</codeblock>
|
| +**
|
| +** except that, for the purposes of auxiliary functions, the query
|
| +** still appears to contain just two phrases - "(first OR 1st)"
|
| +** being treated as a single phrase.
|
| +**
|
| +** <li> By adding multiple synonyms for a single term to the FTS index.
|
| +** Using this method, when tokenizing document text, the tokenizer
|
| +** provides multiple synonyms for each token. So that when a
|
| +** document such as "I won first place" is tokenized, entries are
|
| +** added to the FTS index for "i", "won", "first", "1st" and
|
| +** "place".
|
| +**
|
| +** This way, even if the tokenizer does not provide synonyms
|
| +** when tokenizing query text (it should not - to do would be
|
| +** inefficient), it doesn't matter if the user queries for
|
| +** 'first + place' or '1st + place', as there are entires in the
|
| +** FTS index corresponding to both forms of the first token.
|
| +** </ol>
|
| +**
|
| +** Whether it is parsing document or query text, any call to xToken that
|
| +** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit
|
| +** is considered to supply a synonym for the previous token. For example,
|
| +** when parsing the document "I won first place", a tokenizer that supports
|
| +** synonyms would call xToken() 5 times, as follows:
|
| +**
|
| +** <codeblock>
|
| +** xToken(pCtx, 0, "i", 1, 0, 1);
|
| +** xToken(pCtx, 0, "won", 3, 2, 5);
|
| +** xToken(pCtx, 0, "first", 5, 6, 11);
|
| +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11);
|
| +** xToken(pCtx, 0, "place", 5, 12, 17);
|
| +**</codeblock>
|
| +**
|
| +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time
|
| +** xToken() is called. Multiple synonyms may be specified for a single token
|
| +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence.
|
| +** There is no limit to the number of synonyms that may be provided for a
|
| +** single token.
|
| +**
|
| +** In many cases, method (1) above is the best approach. It does not add
|
| +** extra data to the FTS index or require FTS5 to query for multiple terms,
|
| +** so it is efficient in terms of disk space and query speed. However, it
|
| +** does not support prefix queries very well. If, as suggested above, the
|
| +** token "first" is subsituted for "1st" by the tokenizer, then the query:
|
| +**
|
| +** <codeblock>
|
| +** ... MATCH '1s*'</codeblock>
|
| +**
|
| +** will not match documents that contain the token "1st" (as the tokenizer
|
| +** will probably not map "1s" to any prefix of "first").
|
| +**
|
| +** For full prefix support, method (3) may be preferred. In this case,
|
| +** because the index contains entries for both "first" and "1st", prefix
|
| +** queries such as 'fi*' or '1s*' will match correctly. However, because
|
| +** extra entries are added to the FTS index, this method uses more space
|
| +** within the database.
|
| +**
|
| +** Method (2) offers a midpoint between (1) and (3). Using this method,
|
| +** a query such as '1s*' will match documents that contain the literal
|
| +** token "1st", but not "first" (assuming the tokenizer is not able to
|
| +** provide synonyms for prefixes). However, a non-prefix query like '1st'
|
| +** will match against "1st" and "first". This method does not require
|
| +** extra disk space, as no extra entries are added to the FTS index.
|
| +** On the other hand, it may require more CPU cycles to run MATCH queries,
|
| +** as separate queries of the FTS index are required for each synonym.
|
| +**
|
| +** When using methods (2) or (3), it is important that the tokenizer only
|
| +** provide synonyms when tokenizing document text (method (2)) or query
|
| +** text (method (3)), not both. Doing so will not cause any errors, but is
|
| +** inefficient.
|
| +*/
|
| +typedef struct Fts5Tokenizer Fts5Tokenizer;
|
| +typedef struct fts5_tokenizer fts5_tokenizer;
|
| +struct fts5_tokenizer {
|
| + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
|
| + void (*xDelete)(Fts5Tokenizer*);
|
| + int (*xTokenize)(Fts5Tokenizer*,
|
| + void *pCtx,
|
| + int flags, /* Mask of FTS5_TOKENIZE_* flags */
|
| + const char *pText, int nText,
|
| + int (*xToken)(
|
| + void *pCtx, /* Copy of 2nd argument to xTokenize() */
|
| + int tflags, /* Mask of FTS5_TOKEN_* flags */
|
| + const char *pToken, /* Pointer to buffer containing token */
|
| + int nToken, /* Size of token in bytes */
|
| + int iStart, /* Byte offset of token within input text */
|
| + int iEnd /* Byte offset of end of token within input text */
|
| + )
|
| + );
|
| +};
|
| +
|
| +/* Flags that may be passed as the third argument to xTokenize() */
|
| +#define FTS5_TOKENIZE_QUERY 0x0001
|
| +#define FTS5_TOKENIZE_PREFIX 0x0002
|
| +#define FTS5_TOKENIZE_DOCUMENT 0x0004
|
| +#define FTS5_TOKENIZE_AUX 0x0008
|
| +
|
| +/* Flags that may be passed by the tokenizer implementation back to FTS5
|
| +** as the third argument to the supplied xToken callback. */
|
| +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */
|
| +
|
| +/*
|
| +** END OF CUSTOM TOKENIZERS
|
| +*************************************************************************/
|
| +
|
| +/*************************************************************************
|
| +** FTS5 EXTENSION REGISTRATION API
|
| +*/
|
| +typedef struct fts5_api fts5_api;
|
| +struct fts5_api {
|
| + int iVersion; /* Currently always set to 2 */
|
| +
|
| + /* Create a new tokenizer */
|
| + int (*xCreateTokenizer)(
|
| + fts5_api *pApi,
|
| + const char *zName,
|
| + void *pContext,
|
| + fts5_tokenizer *pTokenizer,
|
| + void (*xDestroy)(void*)
|
| + );
|
| +
|
| + /* Find an existing tokenizer */
|
| + int (*xFindTokenizer)(
|
| + fts5_api *pApi,
|
| + const char *zName,
|
| + void **ppContext,
|
| + fts5_tokenizer *pTokenizer
|
| + );
|
| +
|
| + /* Create a new auxiliary function */
|
| + int (*xCreateFunction)(
|
| + fts5_api *pApi,
|
| + const char *zName,
|
| + void *pContext,
|
| + fts5_extension_function xFunction,
|
| + void (*xDestroy)(void*)
|
| + );
|
| +};
|
| +
|
| +/*
|
| +** END OF REGISTRATION API
|
| +*************************************************************************/
|
| +
|
| +#if 0
|
| +} /* end of the 'extern "C"' block */
|
| +#endif
|
| +
|
| +#endif /* _FTS5_H */
|
| +
|
| +
|
| +/*
|
| +** 2014 May 31
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +*/
|
| +#ifndef _FTS5INT_H
|
| +#define _FTS5INT_H
|
| +
|
| +/* #include "fts5.h" */
|
| +/* #include "sqlite3ext.h" */
|
| +SQLITE_EXTENSION_INIT1
|
| +
|
| +/* #include <string.h> */
|
| +/* #include <assert.h> */
|
| +
|
| +#ifndef SQLITE_AMALGAMATION
|
| +
|
| +typedef unsigned char u8;
|
| +typedef unsigned int u32;
|
| +typedef unsigned short u16;
|
| +typedef sqlite3_int64 i64;
|
| +typedef sqlite3_uint64 u64;
|
| +
|
| +#define ArraySize(x) (sizeof(x) / sizeof(x[0]))
|
| +
|
| +#define testcase(x)
|
| +#define ALWAYS(x) 1
|
| +#define NEVER(x) 0
|
| +
|
| +#define MIN(x,y) (((x) < (y)) ? (x) : (y))
|
| +#define MAX(x,y) (((x) > (y)) ? (x) : (y))
|
| +
|
| +/*
|
| +** Constants for the largest and smallest possible 64-bit signed integers.
|
| +*/
|
| +# define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
|
| +# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
|
| +
|
| +#endif
|
| +
|
| +
|
| +/*
|
| +** Maximum number of prefix indexes on single FTS5 table. This must be
|
| +** less than 32. If it is set to anything large than that, an #error
|
| +** directive in fts5_index.c will cause the build to fail.
|
| +*/
|
| +#define FTS5_MAX_PREFIX_INDEXES 31
|
| +
|
| +#define FTS5_DEFAULT_NEARDIST 10
|
| +#define FTS5_DEFAULT_RANK "bm25"
|
| +
|
| +/* Name of rank and rowid columns */
|
| +#define FTS5_RANK_NAME "rank"
|
| +#define FTS5_ROWID_NAME "rowid"
|
| +
|
| +#ifdef SQLITE_DEBUG
|
| +# define FTS5_CORRUPT sqlite3Fts5Corrupt()
|
| +static int sqlite3Fts5Corrupt(void);
|
| +#else
|
| +# define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
|
| +#endif
|
| +
|
| +/*
|
| +** The assert_nc() macro is similar to the assert() macro, except that it
|
| +** is used for assert() conditions that are true only if it can be
|
| +** guranteed that the database is not corrupt.
|
| +*/
|
| +#ifdef SQLITE_DEBUG
|
| +SQLITE_API extern int sqlite3_fts5_may_be_corrupt;
|
| +# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
|
| +#else
|
| +# define assert_nc(x) assert(x)
|
| +#endif
|
| +
|
| +typedef struct Fts5Global Fts5Global;
|
| +typedef struct Fts5Colset Fts5Colset;
|
| +
|
| +/* If a NEAR() clump or phrase may only match a specific set of columns,
|
| +** then an object of the following type is used to record the set of columns.
|
| +** Each entry in the aiCol[] array is a column that may be matched.
|
| +**
|
| +** This object is used by fts5_expr.c and fts5_index.c.
|
| +*/
|
| +struct Fts5Colset {
|
| + int nCol;
|
| + int aiCol[1];
|
| +};
|
| +
|
| +
|
| +
|
| +/**************************************************************************
|
| +** Interface to code in fts5_config.c. fts5_config.c contains contains code
|
| +** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
|
| +*/
|
| +
|
| +typedef struct Fts5Config Fts5Config;
|
| +
|
| +/*
|
| +** An instance of the following structure encodes all information that can
|
| +** be gleaned from the CREATE VIRTUAL TABLE statement.
|
| +**
|
| +** And all information loaded from the %_config table.
|
| +**
|
| +** nAutomerge:
|
| +** The minimum number of segments that an auto-merge operation should
|
| +** attempt to merge together. A value of 1 sets the object to use the
|
| +** compile time default. Zero disables auto-merge altogether.
|
| +**
|
| +** zContent:
|
| +**
|
| +** zContentRowid:
|
| +** The value of the content_rowid= option, if one was specified. Or
|
| +** the string "rowid" otherwise. This text is not quoted - if it is
|
| +** used as part of an SQL statement it needs to be quoted appropriately.
|
| +**
|
| +** zContentExprlist:
|
| +**
|
| +** pzErrmsg:
|
| +** This exists in order to allow the fts5_index.c module to return a
|
| +** decent error message if it encounters a file-format version it does
|
| +** not understand.
|
| +**
|
| +** bColumnsize:
|
| +** True if the %_docsize table is created.
|
| +**
|
| +** bPrefixIndex:
|
| +** This is only used for debugging. If set to false, any prefix indexes
|
| +** are ignored. This value is configured using:
|
| +**
|
| +** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
|
| +**
|
| +*/
|
| +struct Fts5Config {
|
| + sqlite3 *db; /* Database handle */
|
| + char *zDb; /* Database holding FTS index (e.g. "main") */
|
| + char *zName; /* Name of FTS index */
|
| + int nCol; /* Number of columns */
|
| + char **azCol; /* Column names */
|
| + u8 *abUnindexed; /* True for unindexed columns */
|
| + int nPrefix; /* Number of prefix indexes */
|
| + int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */
|
| + int eContent; /* An FTS5_CONTENT value */
|
| + char *zContent; /* content table */
|
| + char *zContentRowid; /* "content_rowid=" option value */
|
| + int bColumnsize; /* "columnsize=" option value (dflt==1) */
|
| + char *zContentExprlist;
|
| + Fts5Tokenizer *pTok;
|
| + fts5_tokenizer *pTokApi;
|
| +
|
| + /* Values loaded from the %_config table */
|
| + int iCookie; /* Incremented when %_config is modified */
|
| + int pgsz; /* Approximate page size used in %_data */
|
| + int nAutomerge; /* 'automerge' setting */
|
| + int nCrisisMerge; /* Maximum allowed segments per level */
|
| + int nHashSize; /* Bytes of memory for in-memory hash */
|
| + char *zRank; /* Name of rank function */
|
| + char *zRankArgs; /* Arguments to rank function */
|
| +
|
| + /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
|
| + char **pzErrmsg;
|
| +
|
| +#ifdef SQLITE_DEBUG
|
| + int bPrefixIndex; /* True to use prefix-indexes */
|
| +#endif
|
| +};
|
| +
|
| +/* Current expected value of %_config table 'version' field */
|
| +#define FTS5_CURRENT_VERSION 4
|
| +
|
| +#define FTS5_CONTENT_NORMAL 0
|
| +#define FTS5_CONTENT_NONE 1
|
| +#define FTS5_CONTENT_EXTERNAL 2
|
| +
|
| +
|
| +
|
| +
|
| +static int sqlite3Fts5ConfigParse(
|
| + Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
|
| +);
|
| +static void sqlite3Fts5ConfigFree(Fts5Config*);
|
| +
|
| +static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
|
| +
|
| +static int sqlite3Fts5Tokenize(
|
| + Fts5Config *pConfig, /* FTS5 Configuration object */
|
| + int flags, /* FTS5_TOKENIZE_* flags */
|
| + const char *pText, int nText, /* Text to tokenize */
|
| + void *pCtx, /* Context passed to xToken() */
|
| + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
|
| +);
|
| +
|
| +static void sqlite3Fts5Dequote(char *z);
|
| +
|
| +/* Load the contents of the %_config table */
|
| +static int sqlite3Fts5ConfigLoad(Fts5Config*, int);
|
| +
|
| +/* Set the value of a single config attribute */
|
| +static int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
|
| +
|
| +static int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
|
| +
|
| +/*
|
| +** End of interface to code in fts5_config.c.
|
| +**************************************************************************/
|
| +
|
| +/**************************************************************************
|
| +** Interface to code in fts5_buffer.c.
|
| +*/
|
| +
|
| +/*
|
| +** Buffer object for the incremental building of string data.
|
| +*/
|
| +typedef struct Fts5Buffer Fts5Buffer;
|
| +struct Fts5Buffer {
|
| + u8 *p;
|
| + int n;
|
| + int nSpace;
|
| +};
|
| +
|
| +static int sqlite3Fts5BufferSize(int*, Fts5Buffer*, int);
|
| +static void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
|
| +static void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
|
| +static void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
|
| +static void sqlite3Fts5BufferFree(Fts5Buffer*);
|
| +static void sqlite3Fts5BufferZero(Fts5Buffer*);
|
| +static void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
|
| +static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
|
| +
|
| +static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
|
| +
|
| +#define fts5BufferZero(x) sqlite3Fts5BufferZero(x)
|
| +#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
|
| +#define fts5BufferFree(a) sqlite3Fts5BufferFree(a)
|
| +#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
|
| +#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d)
|
| +
|
| +#define fts5BufferGrow(pRc,pBuf,nn) ( \
|
| + (pBuf)->n + (nn) <= (pBuf)->nSpace ? 0 : \
|
| + sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
|
| +)
|
| +
|
| +/* Write and decode big-endian 32-bit integer values */
|
| +static void sqlite3Fts5Put32(u8*, int);
|
| +static int sqlite3Fts5Get32(const u8*);
|
| +
|
| +#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
|
| +#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)
|
| +
|
| +typedef struct Fts5PoslistReader Fts5PoslistReader;
|
| +struct Fts5PoslistReader {
|
| + /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
|
| + const u8 *a; /* Position list to iterate through */
|
| + int n; /* Size of buffer at a[] in bytes */
|
| + int i; /* Current offset in a[] */
|
| +
|
| + u8 bFlag; /* For client use (any custom purpose) */
|
| +
|
| + /* Output variables */
|
| + u8 bEof; /* Set to true at EOF */
|
| + i64 iPos; /* (iCol<<32) + iPos */
|
| +};
|
| +static int sqlite3Fts5PoslistReaderInit(
|
| + const u8 *a, int n, /* Poslist buffer to iterate through */
|
| + Fts5PoslistReader *pIter /* Iterator object to initialize */
|
| +);
|
| +static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
|
| +
|
| +typedef struct Fts5PoslistWriter Fts5PoslistWriter;
|
| +struct Fts5PoslistWriter {
|
| + i64 iPrev;
|
| +};
|
| +static int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
|
| +
|
| +static int sqlite3Fts5PoslistNext64(
|
| + const u8 *a, int n, /* Buffer containing poslist */
|
| + int *pi, /* IN/OUT: Offset within a[] */
|
| + i64 *piOff /* IN/OUT: Current offset */
|
| +);
|
| +
|
| +/* Malloc utility */
|
| +static void *sqlite3Fts5MallocZero(int *pRc, int nByte);
|
| +static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
|
| +
|
| +/* Character set tests (like isspace(), isalpha() etc.) */
|
| +static int sqlite3Fts5IsBareword(char t);
|
| +
|
| +/*
|
| +** End of interface to code in fts5_buffer.c.
|
| +**************************************************************************/
|
| +
|
| +/**************************************************************************
|
| +** Interface to code in fts5_index.c. fts5_index.c contains contains code
|
| +** to access the data stored in the %_data table.
|
| +*/
|
| +
|
| +typedef struct Fts5Index Fts5Index;
|
| +typedef struct Fts5IndexIter Fts5IndexIter;
|
| +
|
| +/*
|
| +** Values used as part of the flags argument passed to IndexQuery().
|
| +*/
|
| +#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */
|
| +#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */
|
| +#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */
|
| +#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */
|
| +
|
| +/*
|
| +** Create/destroy an Fts5Index object.
|
| +*/
|
| +static int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
|
| +static int sqlite3Fts5IndexClose(Fts5Index *p);
|
| +
|
| +/*
|
| +** for(
|
| +** sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter);
|
| +** 0==sqlite3Fts5IterEof(pIter);
|
| +** sqlite3Fts5IterNext(pIter)
|
| +** ){
|
| +** i64 iRowid = sqlite3Fts5IterRowid(pIter);
|
| +** }
|
| +*/
|
| +
|
| +/*
|
| +** Open a new iterator to iterate though all rowids that match the
|
| +** specified token or token prefix.
|
| +*/
|
| +static int sqlite3Fts5IndexQuery(
|
| + Fts5Index *p, /* FTS index to query */
|
| + const char *pToken, int nToken, /* Token (or prefix) to query for */
|
| + int flags, /* Mask of FTS5INDEX_QUERY_X flags */
|
| + Fts5Colset *pColset, /* Match these columns only */
|
| + Fts5IndexIter **ppIter /* OUT: New iterator object */
|
| +);
|
| +
|
| +/*
|
| +** The various operations on open token or token prefix iterators opened
|
| +** using sqlite3Fts5IndexQuery().
|
| +*/
|
| +static int sqlite3Fts5IterEof(Fts5IndexIter*);
|
| +static int sqlite3Fts5IterNext(Fts5IndexIter*);
|
| +static int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
|
| +static i64 sqlite3Fts5IterRowid(Fts5IndexIter*);
|
| +static int sqlite3Fts5IterPoslist(Fts5IndexIter*,Fts5Colset*, const u8**, int*, i64*);
|
| +static int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf);
|
| +
|
| +/*
|
| +** Close an iterator opened by sqlite3Fts5IndexQuery().
|
| +*/
|
| +static void sqlite3Fts5IterClose(Fts5IndexIter*);
|
| +
|
| +/*
|
| +** This interface is used by the fts5vocab module.
|
| +*/
|
| +static const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
|
| +static int sqlite3Fts5IterNextScan(Fts5IndexIter*);
|
| +
|
| +
|
| +/*
|
| +** Insert or remove data to or from the index. Each time a document is
|
| +** added to or removed from the index, this function is called one or more
|
| +** times.
|
| +**
|
| +** For an insert, it must be called once for each token in the new document.
|
| +** If the operation is a delete, it must be called (at least) once for each
|
| +** unique token in the document with an iCol value less than zero. The iPos
|
| +** argument is ignored for a delete.
|
| +*/
|
| +static int sqlite3Fts5IndexWrite(
|
| + Fts5Index *p, /* Index to write to */
|
| + int iCol, /* Column token appears in (-ve -> delete) */
|
| + int iPos, /* Position of token within column */
|
| + const char *pToken, int nToken /* Token to add or remove to or from index */
|
| +);
|
| +
|
| +/*
|
| +** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
|
| +** document iDocid.
|
| +*/
|
| +static int sqlite3Fts5IndexBeginWrite(
|
| + Fts5Index *p, /* Index to write to */
|
| + int bDelete, /* True if current operation is a delete */
|
| + i64 iDocid /* Docid to add or remove data from */
|
| +);
|
| +
|
| +/*
|
| +** Flush any data stored in the in-memory hash tables to the database.
|
| +** If the bCommit flag is true, also close any open blob handles.
|
| +*/
|
| +static int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit);
|
| +
|
| +/*
|
| +** Discard any data stored in the in-memory hash tables. Do not write it
|
| +** to the database. Additionally, assume that the contents of the %_data
|
| +** table may have changed on disk. So any in-memory caches of %_data
|
| +** records must be invalidated.
|
| +*/
|
| +static int sqlite3Fts5IndexRollback(Fts5Index *p);
|
| +
|
| +/*
|
| +** Get or set the "averages" values.
|
| +*/
|
| +static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
|
| +static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
|
| +
|
| +/*
|
| +** Functions called by the storage module as part of integrity-check.
|
| +*/
|
| +static u64 sqlite3Fts5IndexCksum(Fts5Config*,i64,int,int,const char*,int);
|
| +static int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
|
| +
|
| +/*
|
| +** Called during virtual module initialization to register UDF
|
| +** fts5_decode() with SQLite
|
| +*/
|
| +static int sqlite3Fts5IndexInit(sqlite3*);
|
| +
|
| +static int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
|
| +
|
| +/*
|
| +** Return the total number of entries read from the %_data table by
|
| +** this connection since it was created.
|
| +*/
|
| +static int sqlite3Fts5IndexReads(Fts5Index *p);
|
| +
|
| +static int sqlite3Fts5IndexReinit(Fts5Index *p);
|
| +static int sqlite3Fts5IndexOptimize(Fts5Index *p);
|
| +static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
|
| +
|
| +static int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
|
| +
|
| +/*
|
| +** End of interface to code in fts5_index.c.
|
| +**************************************************************************/
|
| +
|
| +/**************************************************************************
|
| +** Interface to code in fts5_varint.c.
|
| +*/
|
| +static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
|
| +static int sqlite3Fts5GetVarintLen(u32 iVal);
|
| +static u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
|
| +static int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
|
| +
|
| +#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
|
| +#define fts5GetVarint sqlite3Fts5GetVarint
|
| +
|
| +#define fts5FastGetVarint32(a, iOff, nVal) { \
|
| + nVal = (a)[iOff++]; \
|
| + if( nVal & 0x80 ){ \
|
| + iOff--; \
|
| + iOff += fts5GetVarint32(&(a)[iOff], nVal); \
|
| + } \
|
| +}
|
| +
|
| +
|
| +/*
|
| +** End of interface to code in fts5_varint.c.
|
| +**************************************************************************/
|
| +
|
| +
|
| +/**************************************************************************
|
| +** Interface to code in fts5.c.
|
| +*/
|
| +
|
| +static int sqlite3Fts5GetTokenizer(
|
| + Fts5Global*,
|
| + const char **azArg,
|
| + int nArg,
|
| + Fts5Tokenizer**,
|
| + fts5_tokenizer**,
|
| + char **pzErr
|
| +);
|
| +
|
| +static Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **);
|
| +
|
| +/*
|
| +** End of interface to code in fts5.c.
|
| +**************************************************************************/
|
| +
|
| +/**************************************************************************
|
| +** Interface to code in fts5_hash.c.
|
| +*/
|
| +typedef struct Fts5Hash Fts5Hash;
|
| +
|
| +/*
|
| +** Create a hash table, free a hash table.
|
| +*/
|
| +static int sqlite3Fts5HashNew(Fts5Hash**, int *pnSize);
|
| +static void sqlite3Fts5HashFree(Fts5Hash*);
|
| +
|
| +static int sqlite3Fts5HashWrite(
|
| + Fts5Hash*,
|
| + i64 iRowid, /* Rowid for this entry */
|
| + int iCol, /* Column token appears in (-ve -> delete) */
|
| + int iPos, /* Position of token within column */
|
| + char bByte,
|
| + const char *pToken, int nToken /* Token to add or remove to or from index */
|
| +);
|
| +
|
| +/*
|
| +** Empty (but do not delete) a hash table.
|
| +*/
|
| +static void sqlite3Fts5HashClear(Fts5Hash*);
|
| +
|
| +static int sqlite3Fts5HashQuery(
|
| + Fts5Hash*, /* Hash table to query */
|
| + const char *pTerm, int nTerm, /* Query term */
|
| + const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */
|
| + int *pnDoclist /* OUT: Size of doclist in bytes */
|
| +);
|
| +
|
| +static int sqlite3Fts5HashScanInit(
|
| + Fts5Hash*, /* Hash table to query */
|
| + const char *pTerm, int nTerm /* Query prefix */
|
| +);
|
| +static void sqlite3Fts5HashScanNext(Fts5Hash*);
|
| +static int sqlite3Fts5HashScanEof(Fts5Hash*);
|
| +static void sqlite3Fts5HashScanEntry(Fts5Hash *,
|
| + const char **pzTerm, /* OUT: term (nul-terminated) */
|
| + const u8 **ppDoclist, /* OUT: pointer to doclist */
|
| + int *pnDoclist /* OUT: size of doclist in bytes */
|
| +);
|
| +
|
| +
|
| +/*
|
| +** End of interface to code in fts5_hash.c.
|
| +**************************************************************************/
|
| +
|
| +/**************************************************************************
|
| +** Interface to code in fts5_storage.c. fts5_storage.c contains contains
|
| +** code to access the data stored in the %_content and %_docsize tables.
|
| +*/
|
| +
|
| +#define FTS5_STMT_SCAN_ASC 0 /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
|
| +#define FTS5_STMT_SCAN_DESC 1 /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
|
| +#define FTS5_STMT_LOOKUP 2 /* SELECT rowid, * FROM ... WHERE rowid=? */
|
| +
|
| +typedef struct Fts5Storage Fts5Storage;
|
| +
|
| +static int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
|
| +static int sqlite3Fts5StorageClose(Fts5Storage *p);
|
| +static int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
|
| +
|
| +static int sqlite3Fts5DropAll(Fts5Config*);
|
| +static int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
|
| +
|
| +static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
|
| +static int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
|
| +static int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
|
| +
|
| +static int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
|
| +
|
| +static int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
|
| +static void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
|
| +
|
| +static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
|
| +static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
|
| +static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
|
| +
|
| +static int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
|
| +static int sqlite3Fts5StorageRollback(Fts5Storage *p);
|
| +
|
| +static int sqlite3Fts5StorageConfigValue(
|
| + Fts5Storage *p, const char*, sqlite3_value*, int
|
| +);
|
| +
|
| +static int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
|
| +
|
| +static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
|
| +static int sqlite3Fts5StorageRebuild(Fts5Storage *p);
|
| +static int sqlite3Fts5StorageOptimize(Fts5Storage *p);
|
| +static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
|
| +
|
| +/*
|
| +** End of interface to code in fts5_storage.c.
|
| +**************************************************************************/
|
| +
|
| +
|
| +/**************************************************************************
|
| +** Interface to code in fts5_expr.c.
|
| +*/
|
| +typedef struct Fts5Expr Fts5Expr;
|
| +typedef struct Fts5ExprNode Fts5ExprNode;
|
| +typedef struct Fts5Parse Fts5Parse;
|
| +typedef struct Fts5Token Fts5Token;
|
| +typedef struct Fts5ExprPhrase Fts5ExprPhrase;
|
| +typedef struct Fts5ExprNearset Fts5ExprNearset;
|
| +
|
| +struct Fts5Token {
|
| + const char *p; /* Token text (not NULL terminated) */
|
| + int n; /* Size of buffer p in bytes */
|
| +};
|
| +
|
| +/* Parse a MATCH expression. */
|
| +static int sqlite3Fts5ExprNew(
|
| + Fts5Config *pConfig,
|
| + const char *zExpr,
|
| + Fts5Expr **ppNew,
|
| + char **pzErr
|
| +);
|
| +
|
| +/*
|
| +** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
|
| +** rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
|
| +** rc = sqlite3Fts5ExprNext(pExpr)
|
| +** ){
|
| +** // The document with rowid iRowid matches the expression!
|
| +** i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
|
| +** }
|
| +*/
|
| +static int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc);
|
| +static int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
|
| +static int sqlite3Fts5ExprEof(Fts5Expr*);
|
| +static i64 sqlite3Fts5ExprRowid(Fts5Expr*);
|
| +
|
| +static void sqlite3Fts5ExprFree(Fts5Expr*);
|
| +
|
| +/* Called during startup to register a UDF with SQLite */
|
| +static int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
|
| +
|
| +static int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
|
| +static int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
|
| +static int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
|
| +
|
| +static int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
|
| +
|
| +/*******************************************
|
| +** The fts5_expr.c API above this point is used by the other hand-written
|
| +** C code in this module. The interfaces below this point are called by
|
| +** the parser code in fts5parse.y. */
|
| +
|
| +static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
|
| +
|
| +static Fts5ExprNode *sqlite3Fts5ParseNode(
|
| + Fts5Parse *pParse,
|
| + int eType,
|
| + Fts5ExprNode *pLeft,
|
| + Fts5ExprNode *pRight,
|
| + Fts5ExprNearset *pNear
|
| +);
|
| +
|
| +static Fts5ExprPhrase *sqlite3Fts5ParseTerm(
|
| + Fts5Parse *pParse,
|
| + Fts5ExprPhrase *pPhrase,
|
| + Fts5Token *pToken,
|
| + int bPrefix
|
| +);
|
| +
|
| +static Fts5ExprNearset *sqlite3Fts5ParseNearset(
|
| + Fts5Parse*,
|
| + Fts5ExprNearset*,
|
| + Fts5ExprPhrase*
|
| +);
|
| +
|
| +static Fts5Colset *sqlite3Fts5ParseColset(
|
| + Fts5Parse*,
|
| + Fts5Colset*,
|
| + Fts5Token *
|
| +);
|
| +
|
| +static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
|
| +static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
|
| +static void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
|
| +
|
| +static void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
|
| +static void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
|
| +static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
|
| +static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
|
| +
|
| +/*
|
| +** End of interface to code in fts5_expr.c.
|
| +**************************************************************************/
|
| +
|
| +
|
| +
|
| +/**************************************************************************
|
| +** Interface to code in fts5_aux.c.
|
| +*/
|
| +
|
| +static int sqlite3Fts5AuxInit(fts5_api*);
|
| +/*
|
| +** End of interface to code in fts5_aux.c.
|
| +**************************************************************************/
|
| +
|
| +/**************************************************************************
|
| +** Interface to code in fts5_tokenizer.c.
|
| +*/
|
| +
|
| +static int sqlite3Fts5TokenizerInit(fts5_api*);
|
| +/*
|
| +** End of interface to code in fts5_tokenizer.c.
|
| +**************************************************************************/
|
| +
|
| +/**************************************************************************
|
| +** Interface to code in fts5_vocab.c.
|
| +*/
|
| +
|
| +static int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
|
| +
|
| +/*
|
| +** End of interface to code in fts5_vocab.c.
|
| +**************************************************************************/
|
| +
|
| +
|
| +/**************************************************************************
|
| +** Interface to automatically generated code in fts5_unicode2.c.
|
| +*/
|
| +static int sqlite3Fts5UnicodeIsalnum(int c);
|
| +static int sqlite3Fts5UnicodeIsdiacritic(int c);
|
| +static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
|
| +/*
|
| +** End of interface to code in fts5_unicode2.c.
|
| +**************************************************************************/
|
| +
|
| +#endif
|
| +
|
| +#define FTS5_OR 1
|
| +#define FTS5_AND 2
|
| +#define FTS5_NOT 3
|
| +#define FTS5_TERM 4
|
| +#define FTS5_COLON 5
|
| +#define FTS5_LP 6
|
| +#define FTS5_RP 7
|
| +#define FTS5_LCP 8
|
| +#define FTS5_RCP 9
|
| +#define FTS5_STRING 10
|
| +#define FTS5_COMMA 11
|
| +#define FTS5_PLUS 12
|
| +#define FTS5_STAR 13
|
| +
|
| +/*
|
| +** 2000-05-29
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +*************************************************************************
|
| +** Driver template for the LEMON parser generator.
|
| +**
|
| +** The "lemon" program processes an LALR(1) input grammar file, then uses
|
| +** this template to construct a parser. The "lemon" program inserts text
|
| +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the
|
| +** interstitial "-" characters) contained in this template is changed into
|
| +** the value of the %name directive from the grammar. Otherwise, the content
|
| +** of this template is copied straight through into the generate parser
|
| +** source file.
|
| +**
|
| +** The following is the concatenation of all %include directives from the
|
| +** input grammar file:
|
| +*/
|
| +/* #include <stdio.h> */
|
| +/************ Begin %include sections from the grammar ************************/
|
| +
|
| +/* #include "fts5Int.h" */
|
| +/* #include "fts5parse.h" */
|
| +
|
| +/*
|
| +** Disable all error recovery processing in the parser push-down
|
| +** automaton.
|
| +*/
|
| +#define fts5YYNOERRORRECOVERY 1
|
| +
|
| +/*
|
| +** Make fts5yytestcase() the same as testcase()
|
| +*/
|
| +#define fts5yytestcase(X) testcase(X)
|
| +
|
| +/*
|
| +** Indicate that sqlite3ParserFree() will never be called with a null
|
| +** pointer.
|
| +*/
|
| +#define fts5YYPARSEFREENOTNULL 1
|
| +
|
| +/*
|
| +** Alternative datatype for the argument to the malloc() routine passed
|
| +** into sqlite3ParserAlloc(). The default is size_t.
|
| +*/
|
| +#define fts5YYMALLOCARGTYPE u64
|
| +
|
| +/**************** End of %include directives **********************************/
|
| +/* These constants specify the various numeric values for terminal symbols
|
| +** in a format understandable to "makeheaders". This section is blank unless
|
| +** "lemon" is run with the "-m" command-line option.
|
| +***************** Begin makeheaders token definitions *************************/
|
| +/**************** End makeheaders token definitions ***************************/
|
| +
|
| +/* The next sections is a series of control #defines.
|
| +** various aspects of the generated parser.
|
| +** fts5YYCODETYPE is the data type used to store the integer codes
|
| +** that represent terminal and non-terminal symbols.
|
| +** "unsigned char" is used if there are fewer than
|
| +** 256 symbols. Larger types otherwise.
|
| +** fts5YYNOCODE is a number of type fts5YYCODETYPE that is not used for
|
| +** any terminal or nonterminal symbol.
|
| +** fts5YYFALLBACK If defined, this indicates that one or more tokens
|
| +** (also known as: "terminal symbols") have fall-back
|
| +** values which should be used if the original symbol
|
| +** would not parse. This permits keywords to sometimes
|
| +** be used as identifiers, for example.
|
| +** fts5YYACTIONTYPE is the data type used for "action codes" - numbers
|
| +** that indicate what to do in response to the next
|
| +** token.
|
| +** sqlite3Fts5ParserFTS5TOKENTYPE is the data type used for minor type for terminal
|
| +** symbols. Background: A "minor type" is a semantic
|
| +** value associated with a terminal or non-terminal
|
| +** symbols. For example, for an "ID" terminal symbol,
|
| +** the minor type might be the name of the identifier.
|
| +** Each non-terminal can have a different minor type.
|
| +** Terminal symbols all have the same minor type, though.
|
| +** This macros defines the minor type for terminal
|
| +** symbols.
|
| +** fts5YYMINORTYPE is the data type used for all minor types.
|
| +** This is typically a union of many types, one of
|
| +** which is sqlite3Fts5ParserFTS5TOKENTYPE. The entry in the union
|
| +** for terminal symbols is called "fts5yy0".
|
| +** fts5YYSTACKDEPTH is the maximum depth of the parser's stack. If
|
| +** zero the stack is dynamically sized using realloc()
|
| +** sqlite3Fts5ParserARG_SDECL A static variable declaration for the %extra_argument
|
| +** sqlite3Fts5ParserARG_PDECL A parameter declaration for the %extra_argument
|
| +** sqlite3Fts5ParserARG_STORE Code to store %extra_argument into fts5yypParser
|
| +** sqlite3Fts5ParserARG_FETCH Code to extract %extra_argument from fts5yypParser
|
| +** fts5YYERRORSYMBOL is the code number of the error symbol. If not
|
| +** defined, then do no error processing.
|
| +** fts5YYNSTATE the combined number of states.
|
| +** fts5YYNRULE the number of rules in the grammar
|
| +** fts5YY_MAX_SHIFT Maximum value for shift actions
|
| +** fts5YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions
|
| +** fts5YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions
|
| +** fts5YY_MIN_REDUCE Maximum value for reduce actions
|
| +** fts5YY_ERROR_ACTION The fts5yy_action[] code for syntax error
|
| +** fts5YY_ACCEPT_ACTION The fts5yy_action[] code for accept
|
| +** fts5YY_NO_ACTION The fts5yy_action[] code for no-op
|
| +*/
|
| +#ifndef INTERFACE
|
| +# define INTERFACE 1
|
| +#endif
|
| +/************* Begin control #defines *****************************************/
|
| +#define fts5YYCODETYPE unsigned char
|
| +#define fts5YYNOCODE 27
|
| +#define fts5YYACTIONTYPE unsigned char
|
| +#define sqlite3Fts5ParserFTS5TOKENTYPE Fts5Token
|
| +typedef union {
|
| + int fts5yyinit;
|
| + sqlite3Fts5ParserFTS5TOKENTYPE fts5yy0;
|
| + Fts5Colset* fts5yy3;
|
| + Fts5ExprPhrase* fts5yy11;
|
| + Fts5ExprNode* fts5yy18;
|
| + int fts5yy20;
|
| + Fts5ExprNearset* fts5yy26;
|
| +} fts5YYMINORTYPE;
|
| +#ifndef fts5YYSTACKDEPTH
|
| +#define fts5YYSTACKDEPTH 100
|
| +#endif
|
| +#define sqlite3Fts5ParserARG_SDECL Fts5Parse *pParse;
|
| +#define sqlite3Fts5ParserARG_PDECL ,Fts5Parse *pParse
|
| +#define sqlite3Fts5ParserARG_FETCH Fts5Parse *pParse = fts5yypParser->pParse
|
| +#define sqlite3Fts5ParserARG_STORE fts5yypParser->pParse = pParse
|
| +#define fts5YYNSTATE 26
|
| +#define fts5YYNRULE 24
|
| +#define fts5YY_MAX_SHIFT 25
|
| +#define fts5YY_MIN_SHIFTREDUCE 40
|
| +#define fts5YY_MAX_SHIFTREDUCE 63
|
| +#define fts5YY_MIN_REDUCE 64
|
| +#define fts5YY_MAX_REDUCE 87
|
| +#define fts5YY_ERROR_ACTION 88
|
| +#define fts5YY_ACCEPT_ACTION 89
|
| +#define fts5YY_NO_ACTION 90
|
| +/************* End control #defines *******************************************/
|
| +
|
| +/* The fts5yyzerominor constant is used to initialize instances of
|
| +** fts5YYMINORTYPE objects to zero. */
|
| +static const fts5YYMINORTYPE fts5yyzerominor = { 0 };
|
| +
|
| +/* Define the fts5yytestcase() macro to be a no-op if is not already defined
|
| +** otherwise.
|
| +**
|
| +** Applications can choose to define fts5yytestcase() in the %include section
|
| +** to a macro that can assist in verifying code coverage. For production
|
| +** code the fts5yytestcase() macro should be turned off. But it is useful
|
| +** for testing.
|
| +*/
|
| +#ifndef fts5yytestcase
|
| +# define fts5yytestcase(X)
|
| +#endif
|
| +
|
| +
|
| +/* Next are the tables used to determine what action to take based on the
|
| +** current state and lookahead token. These tables are used to implement
|
| +** functions that take a state number and lookahead value and return an
|
| +** action integer.
|
| +**
|
| +** Suppose the action integer is N. Then the action is determined as
|
| +** follows
|
| +**
|
| +** 0 <= N <= fts5YY_MAX_SHIFT Shift N. That is, push the lookahead
|
| +** token onto the stack and goto state N.
|
| +**
|
| +** N between fts5YY_MIN_SHIFTREDUCE Shift to an arbitrary state then
|
| +** and fts5YY_MAX_SHIFTREDUCE reduce by rule N-fts5YY_MIN_SHIFTREDUCE.
|
| +**
|
| +** N between fts5YY_MIN_REDUCE Reduce by rule N-fts5YY_MIN_REDUCE
|
| +** and fts5YY_MAX_REDUCE
|
| +
|
| +** N == fts5YY_ERROR_ACTION A syntax error has occurred.
|
| +**
|
| +** N == fts5YY_ACCEPT_ACTION The parser accepts its input.
|
| +**
|
| +** N == fts5YY_NO_ACTION No such action. Denotes unused
|
| +** slots in the fts5yy_action[] table.
|
| +**
|
| +** The action table is constructed as a single large table named fts5yy_action[].
|
| +** Given state S and lookahead X, the action is computed as
|
| +**
|
| +** fts5yy_action[ fts5yy_shift_ofst[S] + X ]
|
| +**
|
| +** If the index value fts5yy_shift_ofst[S]+X is out of range or if the value
|
| +** fts5yy_lookahead[fts5yy_shift_ofst[S]+X] is not equal to X or if fts5yy_shift_ofst[S]
|
| +** is equal to fts5YY_SHIFT_USE_DFLT, it means that the action is not in the table
|
| +** and that fts5yy_default[S] should be used instead.
|
| +**
|
| +** The formula above is for computing the action when the lookahead is
|
| +** a terminal symbol. If the lookahead is a non-terminal (as occurs after
|
| +** a reduce action) then the fts5yy_reduce_ofst[] array is used in place of
|
| +** the fts5yy_shift_ofst[] array and fts5YY_REDUCE_USE_DFLT is used in place of
|
| +** fts5YY_SHIFT_USE_DFLT.
|
| +**
|
| +** The following are the tables generated in this section:
|
| +**
|
| +** fts5yy_action[] A single table containing all actions.
|
| +** fts5yy_lookahead[] A table containing the lookahead for each entry in
|
| +** fts5yy_action. Used to detect hash collisions.
|
| +** fts5yy_shift_ofst[] For each state, the offset into fts5yy_action for
|
| +** shifting terminals.
|
| +** fts5yy_reduce_ofst[] For each state, the offset into fts5yy_action for
|
| +** shifting non-terminals after a reduce.
|
| +** fts5yy_default[] Default action for each state.
|
| +**
|
| +*********** Begin parsing tables **********************************************/
|
| +#define fts5YY_ACTTAB_COUNT (78)
|
| +static const fts5YYACTIONTYPE fts5yy_action[] = {
|
| + /* 0 */ 89, 15, 46, 5, 48, 24, 12, 19, 23, 14,
|
| + /* 10 */ 46, 5, 48, 24, 20, 21, 23, 43, 46, 5,
|
| + /* 20 */ 48, 24, 6, 18, 23, 17, 46, 5, 48, 24,
|
| + /* 30 */ 75, 7, 23, 25, 46, 5, 48, 24, 62, 47,
|
| + /* 40 */ 23, 48, 24, 7, 11, 23, 9, 3, 4, 2,
|
| + /* 50 */ 62, 50, 52, 44, 64, 3, 4, 2, 49, 4,
|
| + /* 60 */ 2, 1, 23, 11, 16, 9, 12, 2, 10, 61,
|
| + /* 70 */ 53, 59, 62, 60, 22, 13, 55, 8,
|
| +};
|
| +static const fts5YYCODETYPE fts5yy_lookahead[] = {
|
| + /* 0 */ 15, 16, 17, 18, 19, 20, 10, 11, 23, 16,
|
| + /* 10 */ 17, 18, 19, 20, 23, 24, 23, 16, 17, 18,
|
| + /* 20 */ 19, 20, 22, 23, 23, 16, 17, 18, 19, 20,
|
| + /* 30 */ 5, 6, 23, 16, 17, 18, 19, 20, 13, 17,
|
| + /* 40 */ 23, 19, 20, 6, 8, 23, 10, 1, 2, 3,
|
| + /* 50 */ 13, 9, 10, 7, 0, 1, 2, 3, 19, 2,
|
| + /* 60 */ 3, 6, 23, 8, 21, 10, 10, 3, 10, 25,
|
| + /* 70 */ 10, 10, 13, 25, 12, 10, 7, 5,
|
| +};
|
| +#define fts5YY_SHIFT_USE_DFLT (-5)
|
| +#define fts5YY_SHIFT_COUNT (25)
|
| +#define fts5YY_SHIFT_MIN (-4)
|
| +#define fts5YY_SHIFT_MAX (72)
|
| +static const signed char fts5yy_shift_ofst[] = {
|
| + /* 0 */ 55, 55, 55, 55, 55, 36, -4, 56, 58, 25,
|
| + /* 10 */ 37, 60, 59, 59, 46, 54, 42, 57, 62, 61,
|
| + /* 20 */ 62, 69, 65, 62, 72, 64,
|
| +};
|
| +#define fts5YY_REDUCE_USE_DFLT (-16)
|
| +#define fts5YY_REDUCE_COUNT (13)
|
| +#define fts5YY_REDUCE_MIN (-15)
|
| +#define fts5YY_REDUCE_MAX (48)
|
| +static const signed char fts5yy_reduce_ofst[] = {
|
| + /* 0 */ -15, -7, 1, 9, 17, 22, -9, 0, 39, 44,
|
| + /* 10 */ 44, 43, 44, 48,
|
| +};
|
| +static const fts5YYACTIONTYPE fts5yy_default[] = {
|
| + /* 0 */ 88, 88, 88, 88, 88, 69, 82, 88, 88, 87,
|
| + /* 10 */ 87, 88, 87, 87, 88, 88, 88, 66, 80, 88,
|
| + /* 20 */ 81, 88, 88, 78, 88, 65,
|
| +};
|
| +/********** End of lemon-generated parsing tables *****************************/
|
| +
|
| +/* The next table maps tokens (terminal symbols) into fallback tokens.
|
| +** If a construct like the following:
|
| +**
|
| +** %fallback ID X Y Z.
|
| +**
|
| +** appears in the grammar, then ID becomes a fallback token for X, Y,
|
| +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser
|
| +** but it does not parse, the type of the token is changed to ID and
|
| +** the parse is retried before an error is thrown.
|
| +**
|
| +** This feature can be used, for example, to cause some keywords in a language
|
| +** to revert to identifiers if they keyword does not apply in the context where
|
| +** it appears.
|
| +*/
|
| +#ifdef fts5YYFALLBACK
|
| +static const fts5YYCODETYPE fts5yyFallback[] = {
|
| +};
|
| +#endif /* fts5YYFALLBACK */
|
| +
|
| +/* The following structure represents a single element of the
|
| +** parser's stack. Information stored includes:
|
| +**
|
| +** + The state number for the parser at this level of the stack.
|
| +**
|
| +** + The value of the token stored at this level of the stack.
|
| +** (In other words, the "major" token.)
|
| +**
|
| +** + The semantic value stored at this level of the stack. This is
|
| +** the information used by the action routines in the grammar.
|
| +** It is sometimes called the "minor" token.
|
| +**
|
| +** After the "shift" half of a SHIFTREDUCE action, the stateno field
|
| +** actually contains the reduce action for the second half of the
|
| +** SHIFTREDUCE.
|
| +*/
|
| +struct fts5yyStackEntry {
|
| + fts5YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */
|
| + fts5YYCODETYPE major; /* The major token value. This is the code
|
| + ** number for the token at this stack level */
|
| + fts5YYMINORTYPE minor; /* The user-supplied minor token value. This
|
| + ** is the value of the token */
|
| +};
|
| +typedef struct fts5yyStackEntry fts5yyStackEntry;
|
| +
|
| +/* The state of the parser is completely contained in an instance of
|
| +** the following structure */
|
| +struct fts5yyParser {
|
| + int fts5yyidx; /* Index of top element in stack */
|
| +#ifdef fts5YYTRACKMAXSTACKDEPTH
|
| + int fts5yyidxMax; /* Maximum value of fts5yyidx */
|
| +#endif
|
| + int fts5yyerrcnt; /* Shifts left before out of the error */
|
| + sqlite3Fts5ParserARG_SDECL /* A place to hold %extra_argument */
|
| +#if fts5YYSTACKDEPTH<=0
|
| + int fts5yystksz; /* Current side of the stack */
|
| + fts5yyStackEntry *fts5yystack; /* The parser's stack */
|
| +#else
|
| + fts5yyStackEntry fts5yystack[fts5YYSTACKDEPTH]; /* The parser's stack */
|
| +#endif
|
| +};
|
| +typedef struct fts5yyParser fts5yyParser;
|
| +
|
| +#ifndef NDEBUG
|
| +/* #include <stdio.h> */
|
| +static FILE *fts5yyTraceFILE = 0;
|
| +static char *fts5yyTracePrompt = 0;
|
| +#endif /* NDEBUG */
|
| +
|
| +#ifndef NDEBUG
|
| +/*
|
| +** Turn parser tracing on by giving a stream to which to write the trace
|
| +** and a prompt to preface each trace message. Tracing is turned off
|
| +** by making either argument NULL
|
| +**
|
| +** Inputs:
|
| +** <ul>
|
| +** <li> A FILE* to which trace output should be written.
|
| +** If NULL, then tracing is turned off.
|
| +** <li> A prefix string written at the beginning of every
|
| +** line of trace output. If NULL, then tracing is
|
| +** turned off.
|
| +** </ul>
|
| +**
|
| +** Outputs:
|
| +** None.
|
| +*/
|
| +static void sqlite3Fts5ParserTrace(FILE *TraceFILE, char *zTracePrompt){
|
| + fts5yyTraceFILE = TraceFILE;
|
| + fts5yyTracePrompt = zTracePrompt;
|
| + if( fts5yyTraceFILE==0 ) fts5yyTracePrompt = 0;
|
| + else if( fts5yyTracePrompt==0 ) fts5yyTraceFILE = 0;
|
| +}
|
| +#endif /* NDEBUG */
|
| +
|
| +#ifndef NDEBUG
|
| +/* For tracing shifts, the names of all terminals and nonterminals
|
| +** are required. The following table supplies these names */
|
| +static const char *const fts5yyTokenName[] = {
|
| + "$", "OR", "AND", "NOT",
|
| + "TERM", "COLON", "LP", "RP",
|
| + "LCP", "RCP", "STRING", "COMMA",
|
| + "PLUS", "STAR", "error", "input",
|
| + "expr", "cnearset", "exprlist", "nearset",
|
| + "colset", "colsetlist", "nearphrases", "phrase",
|
| + "neardist_opt", "star_opt",
|
| +};
|
| +#endif /* NDEBUG */
|
| +
|
| +#ifndef NDEBUG
|
| +/* For tracing reduce actions, the names of all rules are required.
|
| +*/
|
| +static const char *const fts5yyRuleName[] = {
|
| + /* 0 */ "input ::= expr",
|
| + /* 1 */ "expr ::= expr AND expr",
|
| + /* 2 */ "expr ::= expr OR expr",
|
| + /* 3 */ "expr ::= expr NOT expr",
|
| + /* 4 */ "expr ::= LP expr RP",
|
| + /* 5 */ "expr ::= exprlist",
|
| + /* 6 */ "exprlist ::= cnearset",
|
| + /* 7 */ "exprlist ::= exprlist cnearset",
|
| + /* 8 */ "cnearset ::= nearset",
|
| + /* 9 */ "cnearset ::= colset COLON nearset",
|
| + /* 10 */ "colset ::= LCP colsetlist RCP",
|
| + /* 11 */ "colset ::= STRING",
|
| + /* 12 */ "colsetlist ::= colsetlist STRING",
|
| + /* 13 */ "colsetlist ::= STRING",
|
| + /* 14 */ "nearset ::= phrase",
|
| + /* 15 */ "nearset ::= STRING LP nearphrases neardist_opt RP",
|
| + /* 16 */ "nearphrases ::= phrase",
|
| + /* 17 */ "nearphrases ::= nearphrases phrase",
|
| + /* 18 */ "neardist_opt ::=",
|
| + /* 19 */ "neardist_opt ::= COMMA STRING",
|
| + /* 20 */ "phrase ::= phrase PLUS STRING star_opt",
|
| + /* 21 */ "phrase ::= STRING star_opt",
|
| + /* 22 */ "star_opt ::= STAR",
|
| + /* 23 */ "star_opt ::=",
|
| +};
|
| +#endif /* NDEBUG */
|
| +
|
| +
|
| +#if fts5YYSTACKDEPTH<=0
|
| +/*
|
| +** Try to increase the size of the parser stack.
|
| +*/
|
| +static void fts5yyGrowStack(fts5yyParser *p){
|
| + int newSize;
|
| + fts5yyStackEntry *pNew;
|
| +
|
| + newSize = p->fts5yystksz*2 + 100;
|
| + pNew = realloc(p->fts5yystack, newSize*sizeof(pNew[0]));
|
| + if( pNew ){
|
| + p->fts5yystack = pNew;
|
| + p->fts5yystksz = newSize;
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + fprintf(fts5yyTraceFILE,"%sStack grows to %d entries!\n",
|
| + fts5yyTracePrompt, p->fts5yystksz);
|
| + }
|
| +#endif
|
| + }
|
| +}
|
| +#endif
|
| +
|
| +/* Datatype of the argument to the memory allocated passed as the
|
| +** second argument to sqlite3Fts5ParserAlloc() below. This can be changed by
|
| +** putting an appropriate #define in the %include section of the input
|
| +** grammar.
|
| +*/
|
| +#ifndef fts5YYMALLOCARGTYPE
|
| +# define fts5YYMALLOCARGTYPE size_t
|
| +#endif
|
| +
|
| +/*
|
| +** This function allocates a new parser.
|
| +** The only argument is a pointer to a function which works like
|
| +** malloc.
|
| +**
|
| +** Inputs:
|
| +** A pointer to the function used to allocate memory.
|
| +**
|
| +** Outputs:
|
| +** A pointer to a parser. This pointer is used in subsequent calls
|
| +** to sqlite3Fts5Parser and sqlite3Fts5ParserFree.
|
| +*/
|
| +static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(fts5YYMALLOCARGTYPE)){
|
| + fts5yyParser *pParser;
|
| + pParser = (fts5yyParser*)(*mallocProc)( (fts5YYMALLOCARGTYPE)sizeof(fts5yyParser) );
|
| + if( pParser ){
|
| + pParser->fts5yyidx = -1;
|
| +#ifdef fts5YYTRACKMAXSTACKDEPTH
|
| + pParser->fts5yyidxMax = 0;
|
| +#endif
|
| +#if fts5YYSTACKDEPTH<=0
|
| + pParser->fts5yystack = NULL;
|
| + pParser->fts5yystksz = 0;
|
| + fts5yyGrowStack(pParser);
|
| +#endif
|
| + }
|
| + return pParser;
|
| +}
|
| +
|
| +/* The following function deletes the "minor type" or semantic value
|
| +** associated with a symbol. The symbol can be either a terminal
|
| +** or nonterminal. "fts5yymajor" is the symbol code, and "fts5yypminor" is
|
| +** a pointer to the value to be deleted. The code used to do the
|
| +** deletions is derived from the %destructor and/or %token_destructor
|
| +** directives of the input grammar.
|
| +*/
|
| +static void fts5yy_destructor(
|
| + fts5yyParser *fts5yypParser, /* The parser */
|
| + fts5YYCODETYPE fts5yymajor, /* Type code for object to destroy */
|
| + fts5YYMINORTYPE *fts5yypminor /* The object to be destroyed */
|
| +){
|
| + sqlite3Fts5ParserARG_FETCH;
|
| + switch( fts5yymajor ){
|
| + /* Here is inserted the actions which take place when a
|
| + ** terminal or non-terminal is destroyed. This can happen
|
| + ** when the symbol is popped from the stack during a
|
| + ** reduce or during error processing or when a parser is
|
| + ** being destroyed before it is finished parsing.
|
| + **
|
| + ** Note: during a reduce, the only symbols destroyed are those
|
| + ** which appear on the RHS of the rule, but which are *not* used
|
| + ** inside the C code.
|
| + */
|
| +/********* Begin destructor definitions ***************************************/
|
| + case 15: /* input */
|
| +{
|
| + (void)pParse;
|
| +}
|
| + break;
|
| + case 16: /* expr */
|
| + case 17: /* cnearset */
|
| + case 18: /* exprlist */
|
| +{
|
| + sqlite3Fts5ParseNodeFree((fts5yypminor->fts5yy18));
|
| +}
|
| + break;
|
| + case 19: /* nearset */
|
| + case 22: /* nearphrases */
|
| +{
|
| + sqlite3Fts5ParseNearsetFree((fts5yypminor->fts5yy26));
|
| +}
|
| + break;
|
| + case 20: /* colset */
|
| + case 21: /* colsetlist */
|
| +{
|
| + sqlite3_free((fts5yypminor->fts5yy3));
|
| +}
|
| + break;
|
| + case 23: /* phrase */
|
| +{
|
| + sqlite3Fts5ParsePhraseFree((fts5yypminor->fts5yy11));
|
| +}
|
| + break;
|
| +/********* End destructor definitions *****************************************/
|
| + default: break; /* If no destructor action specified: do nothing */
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Pop the parser's stack once.
|
| +**
|
| +** If there is a destructor routine associated with the token which
|
| +** is popped from the stack, then call it.
|
| +*/
|
| +static void fts5yy_pop_parser_stack(fts5yyParser *pParser){
|
| + fts5yyStackEntry *fts5yytos;
|
| + assert( pParser->fts5yyidx>=0 );
|
| + fts5yytos = &pParser->fts5yystack[pParser->fts5yyidx--];
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + fprintf(fts5yyTraceFILE,"%sPopping %s\n",
|
| + fts5yyTracePrompt,
|
| + fts5yyTokenName[fts5yytos->major]);
|
| + }
|
| +#endif
|
| + fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor);
|
| +}
|
| +
|
| +/*
|
| +** Deallocate and destroy a parser. Destructors are called for
|
| +** all stack elements before shutting the parser down.
|
| +**
|
| +** If the fts5YYPARSEFREENEVERNULL macro exists (for example because it
|
| +** is defined in a %include section of the input grammar) then it is
|
| +** assumed that the input pointer is never NULL.
|
| +*/
|
| +static void sqlite3Fts5ParserFree(
|
| + void *p, /* The parser to be deleted */
|
| + void (*freeProc)(void*) /* Function used to reclaim memory */
|
| +){
|
| + fts5yyParser *pParser = (fts5yyParser*)p;
|
| +#ifndef fts5YYPARSEFREENEVERNULL
|
| + if( pParser==0 ) return;
|
| +#endif
|
| + while( pParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(pParser);
|
| +#if fts5YYSTACKDEPTH<=0
|
| + free(pParser->fts5yystack);
|
| +#endif
|
| + (*freeProc)((void*)pParser);
|
| +}
|
| +
|
| +/*
|
| +** Return the peak depth of the stack for a parser.
|
| +*/
|
| +#ifdef fts5YYTRACKMAXSTACKDEPTH
|
| +static int sqlite3Fts5ParserStackPeak(void *p){
|
| + fts5yyParser *pParser = (fts5yyParser*)p;
|
| + return pParser->fts5yyidxMax;
|
| +}
|
| +#endif
|
| +
|
| +/*
|
| +** Find the appropriate action for a parser given the terminal
|
| +** look-ahead token iLookAhead.
|
| +*/
|
| +static int fts5yy_find_shift_action(
|
| + fts5yyParser *pParser, /* The parser */
|
| + fts5YYCODETYPE iLookAhead /* The look-ahead token */
|
| +){
|
| + int i;
|
| + int stateno = pParser->fts5yystack[pParser->fts5yyidx].stateno;
|
| +
|
| + if( stateno>=fts5YY_MIN_REDUCE ) return stateno;
|
| + assert( stateno <= fts5YY_SHIFT_COUNT );
|
| + do{
|
| + i = fts5yy_shift_ofst[stateno];
|
| + if( i==fts5YY_SHIFT_USE_DFLT ) return fts5yy_default[stateno];
|
| + assert( iLookAhead!=fts5YYNOCODE );
|
| + i += iLookAhead;
|
| + if( i<0 || i>=fts5YY_ACTTAB_COUNT || fts5yy_lookahead[i]!=iLookAhead ){
|
| + if( iLookAhead>0 ){
|
| +#ifdef fts5YYFALLBACK
|
| + fts5YYCODETYPE iFallback; /* Fallback token */
|
| + if( iLookAhead<sizeof(fts5yyFallback)/sizeof(fts5yyFallback[0])
|
| + && (iFallback = fts5yyFallback[iLookAhead])!=0 ){
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + fprintf(fts5yyTraceFILE, "%sFALLBACK %s => %s\n",
|
| + fts5yyTracePrompt, fts5yyTokenName[iLookAhead], fts5yyTokenName[iFallback]);
|
| + }
|
| +#endif
|
| + assert( fts5yyFallback[iFallback]==0 ); /* Fallback loop must terminate */
|
| + iLookAhead = iFallback;
|
| + continue;
|
| + }
|
| +#endif
|
| +#ifdef fts5YYWILDCARD
|
| + {
|
| + int j = i - iLookAhead + fts5YYWILDCARD;
|
| + if(
|
| +#if fts5YY_SHIFT_MIN+fts5YYWILDCARD<0
|
| + j>=0 &&
|
| +#endif
|
| +#if fts5YY_SHIFT_MAX+fts5YYWILDCARD>=fts5YY_ACTTAB_COUNT
|
| + j<fts5YY_ACTTAB_COUNT &&
|
| +#endif
|
| + fts5yy_lookahead[j]==fts5YYWILDCARD
|
| + ){
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + fprintf(fts5yyTraceFILE, "%sWILDCARD %s => %s\n",
|
| + fts5yyTracePrompt, fts5yyTokenName[iLookAhead],
|
| + fts5yyTokenName[fts5YYWILDCARD]);
|
| + }
|
| +#endif /* NDEBUG */
|
| + return fts5yy_action[j];
|
| + }
|
| + }
|
| +#endif /* fts5YYWILDCARD */
|
| + }
|
| + return fts5yy_default[stateno];
|
| + }else{
|
| + return fts5yy_action[i];
|
| + }
|
| + }while(1);
|
| +}
|
| +
|
| +/*
|
| +** Find the appropriate action for a parser given the non-terminal
|
| +** look-ahead token iLookAhead.
|
| +*/
|
| +static int fts5yy_find_reduce_action(
|
| + int stateno, /* Current state number */
|
| + fts5YYCODETYPE iLookAhead /* The look-ahead token */
|
| +){
|
| + int i;
|
| +#ifdef fts5YYERRORSYMBOL
|
| + if( stateno>fts5YY_REDUCE_COUNT ){
|
| + return fts5yy_default[stateno];
|
| + }
|
| +#else
|
| + assert( stateno<=fts5YY_REDUCE_COUNT );
|
| +#endif
|
| + i = fts5yy_reduce_ofst[stateno];
|
| + assert( i!=fts5YY_REDUCE_USE_DFLT );
|
| + assert( iLookAhead!=fts5YYNOCODE );
|
| + i += iLookAhead;
|
| +#ifdef fts5YYERRORSYMBOL
|
| + if( i<0 || i>=fts5YY_ACTTAB_COUNT || fts5yy_lookahead[i]!=iLookAhead ){
|
| + return fts5yy_default[stateno];
|
| + }
|
| +#else
|
| + assert( i>=0 && i<fts5YY_ACTTAB_COUNT );
|
| + assert( fts5yy_lookahead[i]==iLookAhead );
|
| +#endif
|
| + return fts5yy_action[i];
|
| +}
|
| +
|
| +/*
|
| +** The following routine is called if the stack overflows.
|
| +*/
|
| +static void fts5yyStackOverflow(fts5yyParser *fts5yypParser, fts5YYMINORTYPE *fts5yypMinor){
|
| + sqlite3Fts5ParserARG_FETCH;
|
| + fts5yypParser->fts5yyidx--;
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + fprintf(fts5yyTraceFILE,"%sStack Overflow!\n",fts5yyTracePrompt);
|
| + }
|
| +#endif
|
| + while( fts5yypParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(fts5yypParser);
|
| + /* Here code is inserted which will execute if the parser
|
| + ** stack every overflows */
|
| +/******** Begin %stack_overflow code ******************************************/
|
| +
|
| + assert( 0 );
|
| +/******** End %stack_overflow code ********************************************/
|
| + sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument var */
|
| +}
|
| +
|
| +/*
|
| +** Print tracing information for a SHIFT action
|
| +*/
|
| +#ifndef NDEBUG
|
| +static void fts5yyTraceShift(fts5yyParser *fts5yypParser, int fts5yyNewState){
|
| + if( fts5yyTraceFILE ){
|
| + if( fts5yyNewState<fts5YYNSTATE ){
|
| + fprintf(fts5yyTraceFILE,"%sShift '%s', go to state %d\n",
|
| + fts5yyTracePrompt,fts5yyTokenName[fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx].major],
|
| + fts5yyNewState);
|
| + }else{
|
| + fprintf(fts5yyTraceFILE,"%sShift '%s'\n",
|
| + fts5yyTracePrompt,fts5yyTokenName[fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx].major]);
|
| + }
|
| + }
|
| +}
|
| +#else
|
| +# define fts5yyTraceShift(X,Y)
|
| +#endif
|
| +
|
| +/*
|
| +** Perform a shift action.
|
| +*/
|
| +static void fts5yy_shift(
|
| + fts5yyParser *fts5yypParser, /* The parser to be shifted */
|
| + int fts5yyNewState, /* The new state to shift in */
|
| + int fts5yyMajor, /* The major token to shift in */
|
| + fts5YYMINORTYPE *fts5yypMinor /* Pointer to the minor token to shift in */
|
| +){
|
| + fts5yyStackEntry *fts5yytos;
|
| + fts5yypParser->fts5yyidx++;
|
| +#ifdef fts5YYTRACKMAXSTACKDEPTH
|
| + if( fts5yypParser->fts5yyidx>fts5yypParser->fts5yyidxMax ){
|
| + fts5yypParser->fts5yyidxMax = fts5yypParser->fts5yyidx;
|
| + }
|
| +#endif
|
| +#if fts5YYSTACKDEPTH>0
|
| + if( fts5yypParser->fts5yyidx>=fts5YYSTACKDEPTH ){
|
| + fts5yyStackOverflow(fts5yypParser, fts5yypMinor);
|
| + return;
|
| + }
|
| +#else
|
| + if( fts5yypParser->fts5yyidx>=fts5yypParser->fts5yystksz ){
|
| + fts5yyGrowStack(fts5yypParser);
|
| + if( fts5yypParser->fts5yyidx>=fts5yypParser->fts5yystksz ){
|
| + fts5yyStackOverflow(fts5yypParser, fts5yypMinor);
|
| + return;
|
| + }
|
| + }
|
| +#endif
|
| + fts5yytos = &fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx];
|
| + fts5yytos->stateno = (fts5YYACTIONTYPE)fts5yyNewState;
|
| + fts5yytos->major = (fts5YYCODETYPE)fts5yyMajor;
|
| + fts5yytos->minor = *fts5yypMinor;
|
| + fts5yyTraceShift(fts5yypParser, fts5yyNewState);
|
| +}
|
| +
|
| +/* The following table contains information about every rule that
|
| +** is used during the reduce.
|
| +*/
|
| +static const struct {
|
| + fts5YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */
|
| + unsigned char nrhs; /* Number of right-hand side symbols in the rule */
|
| +} fts5yyRuleInfo[] = {
|
| + { 15, 1 },
|
| + { 16, 3 },
|
| + { 16, 3 },
|
| + { 16, 3 },
|
| + { 16, 3 },
|
| + { 16, 1 },
|
| + { 18, 1 },
|
| + { 18, 2 },
|
| + { 17, 1 },
|
| + { 17, 3 },
|
| + { 20, 3 },
|
| + { 20, 1 },
|
| + { 21, 2 },
|
| + { 21, 1 },
|
| + { 19, 1 },
|
| + { 19, 5 },
|
| + { 22, 1 },
|
| + { 22, 2 },
|
| + { 24, 0 },
|
| + { 24, 2 },
|
| + { 23, 4 },
|
| + { 23, 2 },
|
| + { 25, 1 },
|
| + { 25, 0 },
|
| +};
|
| +
|
| +static void fts5yy_accept(fts5yyParser*); /* Forward Declaration */
|
| +
|
| +/*
|
| +** Perform a reduce action and the shift that must immediately
|
| +** follow the reduce.
|
| +*/
|
| +static void fts5yy_reduce(
|
| + fts5yyParser *fts5yypParser, /* The parser */
|
| + int fts5yyruleno /* Number of the rule by which to reduce */
|
| +){
|
| + int fts5yygoto; /* The next state */
|
| + int fts5yyact; /* The next action */
|
| + fts5YYMINORTYPE fts5yygotominor; /* The LHS of the rule reduced */
|
| + fts5yyStackEntry *fts5yymsp; /* The top of the parser's stack */
|
| + int fts5yysize; /* Amount to pop the stack */
|
| + sqlite3Fts5ParserARG_FETCH;
|
| + fts5yymsp = &fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx];
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE && fts5yyruleno>=0
|
| + && fts5yyruleno<(int)(sizeof(fts5yyRuleName)/sizeof(fts5yyRuleName[0])) ){
|
| + fts5yysize = fts5yyRuleInfo[fts5yyruleno].nrhs;
|
| + fprintf(fts5yyTraceFILE, "%sReduce [%s], go to state %d.\n", fts5yyTracePrompt,
|
| + fts5yyRuleName[fts5yyruleno], fts5yymsp[-fts5yysize].stateno);
|
| + }
|
| +#endif /* NDEBUG */
|
| + fts5yygotominor = fts5yyzerominor;
|
| +
|
| + switch( fts5yyruleno ){
|
| + /* Beginning here are the reduction cases. A typical example
|
| + ** follows:
|
| + ** case 0:
|
| + ** #line <lineno> <grammarfile>
|
| + ** { ... } // User supplied code
|
| + ** #line <lineno> <thisfile>
|
| + ** break;
|
| + */
|
| +/********** Begin reduce actions **********************************************/
|
| + case 0: /* input ::= expr */
|
| +{ sqlite3Fts5ParseFinished(pParse, fts5yymsp[0].minor.fts5yy18); }
|
| + break;
|
| + case 1: /* expr ::= expr AND expr */
|
| +{
|
| + fts5yygotominor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_AND, fts5yymsp[-2].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18, 0);
|
| +}
|
| + break;
|
| + case 2: /* expr ::= expr OR expr */
|
| +{
|
| + fts5yygotominor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_OR, fts5yymsp[-2].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18, 0);
|
| +}
|
| + break;
|
| + case 3: /* expr ::= expr NOT expr */
|
| +{
|
| + fts5yygotominor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_NOT, fts5yymsp[-2].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18, 0);
|
| +}
|
| + break;
|
| + case 4: /* expr ::= LP expr RP */
|
| +{fts5yygotominor.fts5yy18 = fts5yymsp[-1].minor.fts5yy18;}
|
| + break;
|
| + case 5: /* expr ::= exprlist */
|
| + case 6: /* exprlist ::= cnearset */ fts5yytestcase(fts5yyruleno==6);
|
| +{fts5yygotominor.fts5yy18 = fts5yymsp[0].minor.fts5yy18;}
|
| + break;
|
| + case 7: /* exprlist ::= exprlist cnearset */
|
| +{
|
| + fts5yygotominor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_AND, fts5yymsp[-1].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18, 0);
|
| +}
|
| + break;
|
| + case 8: /* cnearset ::= nearset */
|
| +{
|
| + fts5yygotominor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy26);
|
| +}
|
| + break;
|
| + case 9: /* cnearset ::= colset COLON nearset */
|
| +{
|
| + sqlite3Fts5ParseSetColset(pParse, fts5yymsp[0].minor.fts5yy26, fts5yymsp[-2].minor.fts5yy3);
|
| + fts5yygotominor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy26);
|
| +}
|
| + break;
|
| + case 10: /* colset ::= LCP colsetlist RCP */
|
| +{ fts5yygotominor.fts5yy3 = fts5yymsp[-1].minor.fts5yy3; }
|
| + break;
|
| + case 11: /* colset ::= STRING */
|
| +{
|
| + fts5yygotominor.fts5yy3 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0);
|
| +}
|
| + break;
|
| + case 12: /* colsetlist ::= colsetlist STRING */
|
| +{
|
| + fts5yygotominor.fts5yy3 = sqlite3Fts5ParseColset(pParse, fts5yymsp[-1].minor.fts5yy3, &fts5yymsp[0].minor.fts5yy0); }
|
| + break;
|
| + case 13: /* colsetlist ::= STRING */
|
| +{
|
| + fts5yygotominor.fts5yy3 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0);
|
| +}
|
| + break;
|
| + case 14: /* nearset ::= phrase */
|
| +{ fts5yygotominor.fts5yy26 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy11); }
|
| + break;
|
| + case 15: /* nearset ::= STRING LP nearphrases neardist_opt RP */
|
| +{
|
| + sqlite3Fts5ParseNear(pParse, &fts5yymsp[-4].minor.fts5yy0);
|
| + sqlite3Fts5ParseSetDistance(pParse, fts5yymsp[-2].minor.fts5yy26, &fts5yymsp[-1].minor.fts5yy0);
|
| + fts5yygotominor.fts5yy26 = fts5yymsp[-2].minor.fts5yy26;
|
| +}
|
| + break;
|
| + case 16: /* nearphrases ::= phrase */
|
| +{
|
| + fts5yygotominor.fts5yy26 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy11);
|
| +}
|
| + break;
|
| + case 17: /* nearphrases ::= nearphrases phrase */
|
| +{
|
| + fts5yygotominor.fts5yy26 = sqlite3Fts5ParseNearset(pParse, fts5yymsp[-1].minor.fts5yy26, fts5yymsp[0].minor.fts5yy11);
|
| +}
|
| + break;
|
| + case 18: /* neardist_opt ::= */
|
| +{ fts5yygotominor.fts5yy0.p = 0; fts5yygotominor.fts5yy0.n = 0; }
|
| + break;
|
| + case 19: /* neardist_opt ::= COMMA STRING */
|
| +{ fts5yygotominor.fts5yy0 = fts5yymsp[0].minor.fts5yy0; }
|
| + break;
|
| + case 20: /* phrase ::= phrase PLUS STRING star_opt */
|
| +{
|
| + fts5yygotominor.fts5yy11 = sqlite3Fts5ParseTerm(pParse, fts5yymsp[-3].minor.fts5yy11, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy20);
|
| +}
|
| + break;
|
| + case 21: /* phrase ::= STRING star_opt */
|
| +{
|
| + fts5yygotominor.fts5yy11 = sqlite3Fts5ParseTerm(pParse, 0, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy20);
|
| +}
|
| + break;
|
| + case 22: /* star_opt ::= STAR */
|
| +{ fts5yygotominor.fts5yy20 = 1; }
|
| + break;
|
| + case 23: /* star_opt ::= */
|
| +{ fts5yygotominor.fts5yy20 = 0; }
|
| + break;
|
| + default:
|
| + break;
|
| +/********** End reduce actions ************************************************/
|
| + };
|
| + assert( fts5yyruleno>=0 && fts5yyruleno<sizeof(fts5yyRuleInfo)/sizeof(fts5yyRuleInfo[0]) );
|
| + fts5yygoto = fts5yyRuleInfo[fts5yyruleno].lhs;
|
| + fts5yysize = fts5yyRuleInfo[fts5yyruleno].nrhs;
|
| + fts5yypParser->fts5yyidx -= fts5yysize;
|
| + fts5yyact = fts5yy_find_reduce_action(fts5yymsp[-fts5yysize].stateno,(fts5YYCODETYPE)fts5yygoto);
|
| + if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){
|
| + if( fts5yyact>fts5YY_MAX_SHIFT ) fts5yyact += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE;
|
| + /* If the reduce action popped at least
|
| + ** one element off the stack, then we can push the new element back
|
| + ** onto the stack here, and skip the stack overflow test in fts5yy_shift().
|
| + ** That gives a significant speed improvement. */
|
| + if( fts5yysize ){
|
| + fts5yypParser->fts5yyidx++;
|
| + fts5yymsp -= fts5yysize-1;
|
| + fts5yymsp->stateno = (fts5YYACTIONTYPE)fts5yyact;
|
| + fts5yymsp->major = (fts5YYCODETYPE)fts5yygoto;
|
| + fts5yymsp->minor = fts5yygotominor;
|
| + fts5yyTraceShift(fts5yypParser, fts5yyact);
|
| + }else{
|
| + fts5yy_shift(fts5yypParser,fts5yyact,fts5yygoto,&fts5yygotominor);
|
| + }
|
| + }else{
|
| + assert( fts5yyact == fts5YY_ACCEPT_ACTION );
|
| + fts5yy_accept(fts5yypParser);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** The following code executes when the parse fails
|
| +*/
|
| +#ifndef fts5YYNOERRORRECOVERY
|
| +static void fts5yy_parse_failed(
|
| + fts5yyParser *fts5yypParser /* The parser */
|
| +){
|
| + sqlite3Fts5ParserARG_FETCH;
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + fprintf(fts5yyTraceFILE,"%sFail!\n",fts5yyTracePrompt);
|
| + }
|
| +#endif
|
| + while( fts5yypParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(fts5yypParser);
|
| + /* Here code is inserted which will be executed whenever the
|
| + ** parser fails */
|
| +/************ Begin %parse_failure code ***************************************/
|
| +/************ End %parse_failure code *****************************************/
|
| + sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */
|
| +}
|
| +#endif /* fts5YYNOERRORRECOVERY */
|
| +
|
| +/*
|
| +** The following code executes when a syntax error first occurs.
|
| +*/
|
| +static void fts5yy_syntax_error(
|
| + fts5yyParser *fts5yypParser, /* The parser */
|
| + int fts5yymajor, /* The major type of the error token */
|
| + fts5YYMINORTYPE fts5yyminor /* The minor type of the error token */
|
| +){
|
| + sqlite3Fts5ParserARG_FETCH;
|
| +#define FTS5TOKEN (fts5yyminor.fts5yy0)
|
| +/************ Begin %syntax_error code ****************************************/
|
| +
|
| + sqlite3Fts5ParseError(
|
| + pParse, "fts5: syntax error near \"%.*s\"",FTS5TOKEN.n,FTS5TOKEN.p
|
| + );
|
| +/************ End %syntax_error code ******************************************/
|
| + sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */
|
| +}
|
| +
|
| +/*
|
| +** The following is executed when the parser accepts
|
| +*/
|
| +static void fts5yy_accept(
|
| + fts5yyParser *fts5yypParser /* The parser */
|
| +){
|
| + sqlite3Fts5ParserARG_FETCH;
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + fprintf(fts5yyTraceFILE,"%sAccept!\n",fts5yyTracePrompt);
|
| + }
|
| +#endif
|
| + while( fts5yypParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(fts5yypParser);
|
| + /* Here code is inserted which will be executed whenever the
|
| + ** parser accepts */
|
| +/*********** Begin %parse_accept code *****************************************/
|
| +/*********** End %parse_accept code *******************************************/
|
| + sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */
|
| +}
|
| +
|
| +/* The main parser program.
|
| +** The first argument is a pointer to a structure obtained from
|
| +** "sqlite3Fts5ParserAlloc" which describes the current state of the parser.
|
| +** The second argument is the major token number. The third is
|
| +** the minor token. The fourth optional argument is whatever the
|
| +** user wants (and specified in the grammar) and is available for
|
| +** use by the action routines.
|
| +**
|
| +** Inputs:
|
| +** <ul>
|
| +** <li> A pointer to the parser (an opaque structure.)
|
| +** <li> The major token number.
|
| +** <li> The minor token number.
|
| +** <li> An option argument of a grammar-specified type.
|
| +** </ul>
|
| +**
|
| +** Outputs:
|
| +** None.
|
| +*/
|
| +static void sqlite3Fts5Parser(
|
| + void *fts5yyp, /* The parser */
|
| + int fts5yymajor, /* The major token code number */
|
| + sqlite3Fts5ParserFTS5TOKENTYPE fts5yyminor /* The value for the token */
|
| + sqlite3Fts5ParserARG_PDECL /* Optional %extra_argument parameter */
|
| +){
|
| + fts5YYMINORTYPE fts5yyminorunion;
|
| + int fts5yyact; /* The parser action. */
|
| +#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY)
|
| + int fts5yyendofinput; /* True if we are at the end of input */
|
| +#endif
|
| +#ifdef fts5YYERRORSYMBOL
|
| + int fts5yyerrorhit = 0; /* True if fts5yymajor has invoked an error */
|
| +#endif
|
| + fts5yyParser *fts5yypParser; /* The parser */
|
| +
|
| + /* (re)initialize the parser, if necessary */
|
| + fts5yypParser = (fts5yyParser*)fts5yyp;
|
| + if( fts5yypParser->fts5yyidx<0 ){
|
| +#if fts5YYSTACKDEPTH<=0
|
| + if( fts5yypParser->fts5yystksz <=0 ){
|
| + /*memset(&fts5yyminorunion, 0, sizeof(fts5yyminorunion));*/
|
| + fts5yyminorunion = fts5yyzerominor;
|
| + fts5yyStackOverflow(fts5yypParser, &fts5yyminorunion);
|
| + return;
|
| + }
|
| +#endif
|
| + fts5yypParser->fts5yyidx = 0;
|
| + fts5yypParser->fts5yyerrcnt = -1;
|
| + fts5yypParser->fts5yystack[0].stateno = 0;
|
| + fts5yypParser->fts5yystack[0].major = 0;
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + fprintf(fts5yyTraceFILE,"%sInitialize. Empty stack. State 0\n",
|
| + fts5yyTracePrompt);
|
| + }
|
| +#endif
|
| + }
|
| + fts5yyminorunion.fts5yy0 = fts5yyminor;
|
| +#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY)
|
| + fts5yyendofinput = (fts5yymajor==0);
|
| +#endif
|
| + sqlite3Fts5ParserARG_STORE;
|
| +
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + fprintf(fts5yyTraceFILE,"%sInput '%s'\n",fts5yyTracePrompt,fts5yyTokenName[fts5yymajor]);
|
| + }
|
| +#endif
|
| +
|
| + do{
|
| + fts5yyact = fts5yy_find_shift_action(fts5yypParser,(fts5YYCODETYPE)fts5yymajor);
|
| + if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){
|
| + if( fts5yyact > fts5YY_MAX_SHIFT ) fts5yyact += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE;
|
| + fts5yy_shift(fts5yypParser,fts5yyact,fts5yymajor,&fts5yyminorunion);
|
| + fts5yypParser->fts5yyerrcnt--;
|
| + fts5yymajor = fts5YYNOCODE;
|
| + }else if( fts5yyact <= fts5YY_MAX_REDUCE ){
|
| + fts5yy_reduce(fts5yypParser,fts5yyact-fts5YY_MIN_REDUCE);
|
| + }else{
|
| + assert( fts5yyact == fts5YY_ERROR_ACTION );
|
| +#ifdef fts5YYERRORSYMBOL
|
| + int fts5yymx;
|
| +#endif
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + fprintf(fts5yyTraceFILE,"%sSyntax Error!\n",fts5yyTracePrompt);
|
| + }
|
| +#endif
|
| +#ifdef fts5YYERRORSYMBOL
|
| + /* A syntax error has occurred.
|
| + ** The response to an error depends upon whether or not the
|
| + ** grammar defines an error token "ERROR".
|
| + **
|
| + ** This is what we do if the grammar does define ERROR:
|
| + **
|
| + ** * Call the %syntax_error function.
|
| + **
|
| + ** * Begin popping the stack until we enter a state where
|
| + ** it is legal to shift the error symbol, then shift
|
| + ** the error symbol.
|
| + **
|
| + ** * Set the error count to three.
|
| + **
|
| + ** * Begin accepting and shifting new tokens. No new error
|
| + ** processing will occur until three tokens have been
|
| + ** shifted successfully.
|
| + **
|
| + */
|
| + if( fts5yypParser->fts5yyerrcnt<0 ){
|
| + fts5yy_syntax_error(fts5yypParser,fts5yymajor,fts5yyminorunion);
|
| + }
|
| + fts5yymx = fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx].major;
|
| + if( fts5yymx==fts5YYERRORSYMBOL || fts5yyerrorhit ){
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + fprintf(fts5yyTraceFILE,"%sDiscard input token %s\n",
|
| + fts5yyTracePrompt,fts5yyTokenName[fts5yymajor]);
|
| + }
|
| +#endif
|
| + fts5yy_destructor(fts5yypParser, (fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion);
|
| + fts5yymajor = fts5YYNOCODE;
|
| + }else{
|
| + while(
|
| + fts5yypParser->fts5yyidx >= 0 &&
|
| + fts5yymx != fts5YYERRORSYMBOL &&
|
| + (fts5yyact = fts5yy_find_reduce_action(
|
| + fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx].stateno,
|
| + fts5YYERRORSYMBOL)) >= fts5YY_MIN_REDUCE
|
| + ){
|
| + fts5yy_pop_parser_stack(fts5yypParser);
|
| + }
|
| + if( fts5yypParser->fts5yyidx < 0 || fts5yymajor==0 ){
|
| + fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion);
|
| + fts5yy_parse_failed(fts5yypParser);
|
| + fts5yymajor = fts5YYNOCODE;
|
| + }else if( fts5yymx!=fts5YYERRORSYMBOL ){
|
| + fts5YYMINORTYPE u2;
|
| + u2.fts5YYERRSYMDT = 0;
|
| + fts5yy_shift(fts5yypParser,fts5yyact,fts5YYERRORSYMBOL,&u2);
|
| + }
|
| + }
|
| + fts5yypParser->fts5yyerrcnt = 3;
|
| + fts5yyerrorhit = 1;
|
| +#elif defined(fts5YYNOERRORRECOVERY)
|
| + /* If the fts5YYNOERRORRECOVERY macro is defined, then do not attempt to
|
| + ** do any kind of error recovery. Instead, simply invoke the syntax
|
| + ** error routine and continue going as if nothing had happened.
|
| + **
|
| + ** Applications can set this macro (for example inside %include) if
|
| + ** they intend to abandon the parse upon the first syntax error seen.
|
| + */
|
| + fts5yy_syntax_error(fts5yypParser,fts5yymajor,fts5yyminorunion);
|
| + fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion);
|
| + fts5yymajor = fts5YYNOCODE;
|
| +
|
| +#else /* fts5YYERRORSYMBOL is not defined */
|
| + /* This is what we do if the grammar does not define ERROR:
|
| + **
|
| + ** * Report an error message, and throw away the input token.
|
| + **
|
| + ** * If the input token is $, then fail the parse.
|
| + **
|
| + ** As before, subsequent error messages are suppressed until
|
| + ** three input tokens have been successfully shifted.
|
| + */
|
| + if( fts5yypParser->fts5yyerrcnt<=0 ){
|
| + fts5yy_syntax_error(fts5yypParser,fts5yymajor,fts5yyminorunion);
|
| + }
|
| + fts5yypParser->fts5yyerrcnt = 3;
|
| + fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion);
|
| + if( fts5yyendofinput ){
|
| + fts5yy_parse_failed(fts5yypParser);
|
| + }
|
| + fts5yymajor = fts5YYNOCODE;
|
| +#endif
|
| + }
|
| + }while( fts5yymajor!=fts5YYNOCODE && fts5yypParser->fts5yyidx>=0 );
|
| +#ifndef NDEBUG
|
| + if( fts5yyTraceFILE ){
|
| + int i;
|
| + fprintf(fts5yyTraceFILE,"%sReturn. Stack=",fts5yyTracePrompt);
|
| + for(i=1; i<=fts5yypParser->fts5yyidx; i++)
|
| + fprintf(fts5yyTraceFILE,"%c%s", i==1 ? '[' : ' ',
|
| + fts5yyTokenName[fts5yypParser->fts5yystack[i].major]);
|
| + fprintf(fts5yyTraceFILE,"]\n");
|
| + }
|
| +#endif
|
| + return;
|
| +}
|
| +
|
| +/*
|
| +** 2014 May 31
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +*/
|
| +
|
| +
|
| +/* #include "fts5Int.h" */
|
| +#include <math.h> /* amalgamator: keep */
|
| +
|
| +/*
|
| +** Object used to iterate through all "coalesced phrase instances" in
|
| +** a single column of the current row. If the phrase instances in the
|
| +** column being considered do not overlap, this object simply iterates
|
| +** through them. Or, if they do overlap (share one or more tokens in
|
| +** common), each set of overlapping instances is treated as a single
|
| +** match. See documentation for the highlight() auxiliary function for
|
| +** details.
|
| +**
|
| +** Usage is:
|
| +**
|
| +** for(rc = fts5CInstIterNext(pApi, pFts, iCol, &iter);
|
| +** (rc==SQLITE_OK && 0==fts5CInstIterEof(&iter);
|
| +** rc = fts5CInstIterNext(&iter)
|
| +** ){
|
| +** printf("instance starts at %d, ends at %d\n", iter.iStart, iter.iEnd);
|
| +** }
|
| +**
|
| +*/
|
| +typedef struct CInstIter CInstIter;
|
| +struct CInstIter {
|
| + const Fts5ExtensionApi *pApi; /* API offered by current FTS version */
|
| + Fts5Context *pFts; /* First arg to pass to pApi functions */
|
| + int iCol; /* Column to search */
|
| + int iInst; /* Next phrase instance index */
|
| + int nInst; /* Total number of phrase instances */
|
| +
|
| + /* Output variables */
|
| + int iStart; /* First token in coalesced phrase instance */
|
| + int iEnd; /* Last token in coalesced phrase instance */
|
| +};
|
| +
|
| +/*
|
| +** Advance the iterator to the next coalesced phrase instance. Return
|
| +** an SQLite error code if an error occurs, or SQLITE_OK otherwise.
|
| +*/
|
| +static int fts5CInstIterNext(CInstIter *pIter){
|
| + int rc = SQLITE_OK;
|
| + pIter->iStart = -1;
|
| + pIter->iEnd = -1;
|
| +
|
| + while( rc==SQLITE_OK && pIter->iInst<pIter->nInst ){
|
| + int ip; int ic; int io;
|
| + rc = pIter->pApi->xInst(pIter->pFts, pIter->iInst, &ip, &ic, &io);
|
| + if( rc==SQLITE_OK ){
|
| + if( ic==pIter->iCol ){
|
| + int iEnd = io - 1 + pIter->pApi->xPhraseSize(pIter->pFts, ip);
|
| + if( pIter->iStart<0 ){
|
| + pIter->iStart = io;
|
| + pIter->iEnd = iEnd;
|
| + }else if( io<=pIter->iEnd ){
|
| + if( iEnd>pIter->iEnd ) pIter->iEnd = iEnd;
|
| + }else{
|
| + break;
|
| + }
|
| + }
|
| + pIter->iInst++;
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Initialize the iterator object indicated by the final parameter to
|
| +** iterate through coalesced phrase instances in column iCol.
|
| +*/
|
| +static int fts5CInstIterInit(
|
| + const Fts5ExtensionApi *pApi,
|
| + Fts5Context *pFts,
|
| + int iCol,
|
| + CInstIter *pIter
|
| +){
|
| + int rc;
|
| +
|
| + memset(pIter, 0, sizeof(CInstIter));
|
| + pIter->pApi = pApi;
|
| + pIter->pFts = pFts;
|
| + pIter->iCol = iCol;
|
| + rc = pApi->xInstCount(pFts, &pIter->nInst);
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5CInstIterNext(pIter);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +
|
| +/*************************************************************************
|
| +** Start of highlight() implementation.
|
| +*/
|
| +typedef struct HighlightContext HighlightContext;
|
| +struct HighlightContext {
|
| + CInstIter iter; /* Coalesced Instance Iterator */
|
| + int iPos; /* Current token offset in zIn[] */
|
| + int iRangeStart; /* First token to include */
|
| + int iRangeEnd; /* If non-zero, last token to include */
|
| + const char *zOpen; /* Opening highlight */
|
| + const char *zClose; /* Closing highlight */
|
| + const char *zIn; /* Input text */
|
| + int nIn; /* Size of input text in bytes */
|
| + int iOff; /* Current offset within zIn[] */
|
| + char *zOut; /* Output value */
|
| +};
|
| +
|
| +/*
|
| +** Append text to the HighlightContext output string - p->zOut. Argument
|
| +** z points to a buffer containing n bytes of text to append. If n is
|
| +** negative, everything up until the first '\0' is appended to the output.
|
| +**
|
| +** If *pRc is set to any value other than SQLITE_OK when this function is
|
| +** called, it is a no-op. If an error (i.e. an OOM condition) is encountered,
|
| +** *pRc is set to an error code before returning.
|
| +*/
|
| +static void fts5HighlightAppend(
|
| + int *pRc,
|
| + HighlightContext *p,
|
| + const char *z, int n
|
| +){
|
| + if( *pRc==SQLITE_OK ){
|
| + if( n<0 ) n = (int)strlen(z);
|
| + p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
|
| + if( p->zOut==0 ) *pRc = SQLITE_NOMEM;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Tokenizer callback used by implementation of highlight() function.
|
| +*/
|
| +static int fts5HighlightCb(
|
| + void *pContext, /* Pointer to HighlightContext object */
|
| + int tflags, /* Mask of FTS5_TOKEN_* flags */
|
| + const char *pToken, /* Buffer containing token */
|
| + int nToken, /* Size of token in bytes */
|
| + int iStartOff, /* Start offset of token */
|
| + int iEndOff /* End offset of token */
|
| +){
|
| + HighlightContext *p = (HighlightContext*)pContext;
|
| + int rc = SQLITE_OK;
|
| + int iPos;
|
| +
|
| + if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK;
|
| + iPos = p->iPos++;
|
| +
|
| + if( p->iRangeEnd>0 ){
|
| + if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK;
|
| + if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
|
| + }
|
| +
|
| + if( iPos==p->iter.iStart ){
|
| + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
|
| + fts5HighlightAppend(&rc, p, p->zOpen, -1);
|
| + p->iOff = iStartOff;
|
| + }
|
| +
|
| + if( iPos==p->iter.iEnd ){
|
| + if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){
|
| + fts5HighlightAppend(&rc, p, p->zOpen, -1);
|
| + }
|
| + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
|
| + fts5HighlightAppend(&rc, p, p->zClose, -1);
|
| + p->iOff = iEndOff;
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5CInstIterNext(&p->iter);
|
| + }
|
| + }
|
| +
|
| + if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){
|
| + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
|
| + p->iOff = iEndOff;
|
| + if( iPos<p->iter.iEnd ){
|
| + fts5HighlightAppend(&rc, p, p->zClose, -1);
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Implementation of highlight() function.
|
| +*/
|
| +static void fts5HighlightFunction(
|
| + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
|
| + Fts5Context *pFts, /* First arg to pass to pApi functions */
|
| + sqlite3_context *pCtx, /* Context for returning result/error */
|
| + int nVal, /* Number of values in apVal[] array */
|
| + sqlite3_value **apVal /* Array of trailing arguments */
|
| +){
|
| + HighlightContext ctx;
|
| + int rc;
|
| + int iCol;
|
| +
|
| + if( nVal!=3 ){
|
| + const char *zErr = "wrong number of arguments to function highlight()";
|
| + sqlite3_result_error(pCtx, zErr, -1);
|
| + return;
|
| + }
|
| +
|
| + iCol = sqlite3_value_int(apVal[0]);
|
| + memset(&ctx, 0, sizeof(HighlightContext));
|
| + ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
|
| + ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
|
| + rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);
|
| +
|
| + if( ctx.zIn ){
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
|
| + }
|
| + fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
|
| + }
|
| + sqlite3_free(ctx.zOut);
|
| + }
|
| + if( rc!=SQLITE_OK ){
|
| + sqlite3_result_error_code(pCtx, rc);
|
| + }
|
| +}
|
| +/*
|
| +** End of highlight() implementation.
|
| +**************************************************************************/
|
| +
|
| +/*
|
| +** Implementation of snippet() function.
|
| +*/
|
| +static void fts5SnippetFunction(
|
| + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
|
| + Fts5Context *pFts, /* First arg to pass to pApi functions */
|
| + sqlite3_context *pCtx, /* Context for returning result/error */
|
| + int nVal, /* Number of values in apVal[] array */
|
| + sqlite3_value **apVal /* Array of trailing arguments */
|
| +){
|
| + HighlightContext ctx;
|
| + int rc = SQLITE_OK; /* Return code */
|
| + int iCol; /* 1st argument to snippet() */
|
| + const char *zEllips; /* 4th argument to snippet() */
|
| + int nToken; /* 5th argument to snippet() */
|
| + int nInst = 0; /* Number of instance matches this row */
|
| + int i; /* Used to iterate through instances */
|
| + int nPhrase; /* Number of phrases in query */
|
| + unsigned char *aSeen; /* Array of "seen instance" flags */
|
| + int iBestCol; /* Column containing best snippet */
|
| + int iBestStart = 0; /* First token of best snippet */
|
| + int iBestLast; /* Last token of best snippet */
|
| + int nBestScore = 0; /* Score of best snippet */
|
| + int nColSize = 0; /* Total size of iBestCol in tokens */
|
| +
|
| + if( nVal!=5 ){
|
| + const char *zErr = "wrong number of arguments to function snippet()";
|
| + sqlite3_result_error(pCtx, zErr, -1);
|
| + return;
|
| + }
|
| +
|
| + memset(&ctx, 0, sizeof(HighlightContext));
|
| + iCol = sqlite3_value_int(apVal[0]);
|
| + ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
|
| + ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
|
| + zEllips = (const char*)sqlite3_value_text(apVal[3]);
|
| + nToken = sqlite3_value_int(apVal[4]);
|
| + iBestLast = nToken-1;
|
| +
|
| + iBestCol = (iCol>=0 ? iCol : 0);
|
| + nPhrase = pApi->xPhraseCount(pFts);
|
| + aSeen = sqlite3_malloc(nPhrase);
|
| + if( aSeen==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = pApi->xInstCount(pFts, &nInst);
|
| + }
|
| + for(i=0; rc==SQLITE_OK && i<nInst; i++){
|
| + int ip, iSnippetCol, iStart;
|
| + memset(aSeen, 0, nPhrase);
|
| + rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart);
|
| + if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){
|
| + int nScore = 1000;
|
| + int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip);
|
| + int j;
|
| + aSeen[ip] = 1;
|
| +
|
| + for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
|
| + int ic; int io; int iFinal;
|
| + rc = pApi->xInst(pFts, j, &ip, &ic, &io);
|
| + iFinal = io + pApi->xPhraseSize(pFts, ip) - 1;
|
| + if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){
|
| + nScore += aSeen[ip] ? 1000 : 1;
|
| + aSeen[ip] = 1;
|
| + if( iFinal>iLast ) iLast = iFinal;
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK && nScore>nBestScore ){
|
| + iBestCol = iSnippetCol;
|
| + iBestStart = iStart;
|
| + iBestLast = iLast;
|
| + nBestScore = nScore;
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
|
| + }
|
| + if( ctx.zIn ){
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
|
| + }
|
| +
|
| + if( (iBestStart+nToken-1)>iBestLast ){
|
| + iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
|
| + }
|
| + if( iBestStart+nToken>nColSize ){
|
| + iBestStart = nColSize - nToken;
|
| + }
|
| + if( iBestStart<0 ) iBestStart = 0;
|
| +
|
| + ctx.iRangeStart = iBestStart;
|
| + ctx.iRangeEnd = iBestStart + nToken - 1;
|
| +
|
| + if( iBestStart>0 ){
|
| + fts5HighlightAppend(&rc, &ctx, zEllips, -1);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
|
| + }
|
| + if( ctx.iRangeEnd>=(nColSize-1) ){
|
| + fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
|
| + }else{
|
| + fts5HighlightAppend(&rc, &ctx, zEllips, -1);
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
|
| + }else{
|
| + sqlite3_result_error_code(pCtx, rc);
|
| + }
|
| + sqlite3_free(ctx.zOut);
|
| + }
|
| + sqlite3_free(aSeen);
|
| +}
|
| +
|
| +/************************************************************************/
|
| +
|
| +/*
|
| +** The first time the bm25() function is called for a query, an instance
|
| +** of the following structure is allocated and populated.
|
| +*/
|
| +typedef struct Fts5Bm25Data Fts5Bm25Data;
|
| +struct Fts5Bm25Data {
|
| + int nPhrase; /* Number of phrases in query */
|
| + double avgdl; /* Average number of tokens in each row */
|
| + double *aIDF; /* IDF for each phrase */
|
| + double *aFreq; /* Array used to calculate phrase freq. */
|
| +};
|
| +
|
| +/*
|
| +** Callback used by fts5Bm25GetData() to count the number of rows in the
|
| +** table matched by each individual phrase within the query.
|
| +*/
|
| +static int fts5CountCb(
|
| + const Fts5ExtensionApi *pApi,
|
| + Fts5Context *pFts,
|
| + void *pUserData /* Pointer to sqlite3_int64 variable */
|
| +){
|
| + sqlite3_int64 *pn = (sqlite3_int64*)pUserData;
|
| + (*pn)++;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Set *ppData to point to the Fts5Bm25Data object for the current query.
|
| +** If the object has not already been allocated, allocate and populate it
|
| +** now.
|
| +*/
|
| +static int fts5Bm25GetData(
|
| + const Fts5ExtensionApi *pApi,
|
| + Fts5Context *pFts,
|
| + Fts5Bm25Data **ppData /* OUT: bm25-data object for this query */
|
| +){
|
| + int rc = SQLITE_OK; /* Return code */
|
| + Fts5Bm25Data *p; /* Object to return */
|
| +
|
| + p = pApi->xGetAuxdata(pFts, 0);
|
| + if( p==0 ){
|
| + int nPhrase; /* Number of phrases in query */
|
| + sqlite3_int64 nRow = 0; /* Number of rows in table */
|
| + sqlite3_int64 nToken = 0; /* Number of tokens in table */
|
| + int nByte; /* Bytes of space to allocate */
|
| + int i;
|
| +
|
| + /* Allocate the Fts5Bm25Data object */
|
| + nPhrase = pApi->xPhraseCount(pFts);
|
| + nByte = sizeof(Fts5Bm25Data) + nPhrase*2*sizeof(double);
|
| + p = (Fts5Bm25Data*)sqlite3_malloc(nByte);
|
| + if( p==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + memset(p, 0, nByte);
|
| + p->nPhrase = nPhrase;
|
| + p->aIDF = (double*)&p[1];
|
| + p->aFreq = &p->aIDF[nPhrase];
|
| + }
|
| +
|
| + /* Calculate the average document length for this FTS5 table */
|
| + if( rc==SQLITE_OK ) rc = pApi->xRowCount(pFts, &nRow);
|
| + if( rc==SQLITE_OK ) rc = pApi->xColumnTotalSize(pFts, -1, &nToken);
|
| + if( rc==SQLITE_OK ) p->avgdl = (double)nToken / (double)nRow;
|
| +
|
| + /* Calculate an IDF for each phrase in the query */
|
| + for(i=0; rc==SQLITE_OK && i<nPhrase; i++){
|
| + sqlite3_int64 nHit = 0;
|
| + rc = pApi->xQueryPhrase(pFts, i, (void*)&nHit, fts5CountCb);
|
| + if( rc==SQLITE_OK ){
|
| + /* Calculate the IDF (Inverse Document Frequency) for phrase i.
|
| + ** This is done using the standard BM25 formula as found on wikipedia:
|
| + **
|
| + ** IDF = log( (N - nHit + 0.5) / (nHit + 0.5) )
|
| + **
|
| + ** where "N" is the total number of documents in the set and nHit
|
| + ** is the number that contain at least one instance of the phrase
|
| + ** under consideration.
|
| + **
|
| + ** The problem with this is that if (N < 2*nHit), the IDF is
|
| + ** negative. Which is undesirable. So the mimimum allowable IDF is
|
| + ** (1e-6) - roughly the same as a term that appears in just over
|
| + ** half of set of 5,000,000 documents. */
|
| + double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) );
|
| + if( idf<=0.0 ) idf = 1e-6;
|
| + p->aIDF[i] = idf;
|
| + }
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + sqlite3_free(p);
|
| + }else{
|
| + rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
|
| + }
|
| + if( rc!=SQLITE_OK ) p = 0;
|
| + }
|
| + *ppData = p;
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Implementation of bm25() function.
|
| +*/
|
| +static void fts5Bm25Function(
|
| + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
|
| + Fts5Context *pFts, /* First arg to pass to pApi functions */
|
| + sqlite3_context *pCtx, /* Context for returning result/error */
|
| + int nVal, /* Number of values in apVal[] array */
|
| + sqlite3_value **apVal /* Array of trailing arguments */
|
| +){
|
| + const double k1 = 1.2; /* Constant "k1" from BM25 formula */
|
| + const double b = 0.75; /* Constant "b" from BM25 formula */
|
| + int rc = SQLITE_OK; /* Error code */
|
| + double score = 0.0; /* SQL function return value */
|
| + Fts5Bm25Data *pData; /* Values allocated/calculated once only */
|
| + int i; /* Iterator variable */
|
| + int nInst = 0; /* Value returned by xInstCount() */
|
| + double D = 0.0; /* Total number of tokens in row */
|
| + double *aFreq = 0; /* Array of phrase freq. for current row */
|
| +
|
| + /* Calculate the phrase frequency (symbol "f(qi,D)" in the documentation)
|
| + ** for each phrase in the query for the current row. */
|
| + rc = fts5Bm25GetData(pApi, pFts, &pData);
|
| + if( rc==SQLITE_OK ){
|
| + aFreq = pData->aFreq;
|
| + memset(aFreq, 0, sizeof(double) * pData->nPhrase);
|
| + rc = pApi->xInstCount(pFts, &nInst);
|
| + }
|
| + for(i=0; rc==SQLITE_OK && i<nInst; i++){
|
| + int ip; int ic; int io;
|
| + rc = pApi->xInst(pFts, i, &ip, &ic, &io);
|
| + if( rc==SQLITE_OK ){
|
| + double w = (nVal > ic) ? sqlite3_value_double(apVal[ic]) : 1.0;
|
| + aFreq[ip] += w;
|
| + }
|
| + }
|
| +
|
| + /* Figure out the total size of the current row in tokens. */
|
| + if( rc==SQLITE_OK ){
|
| + int nTok;
|
| + rc = pApi->xColumnSize(pFts, -1, &nTok);
|
| + D = (double)nTok;
|
| + }
|
| +
|
| + /* Determine the BM25 score for the current row. */
|
| + for(i=0; rc==SQLITE_OK && i<pData->nPhrase; i++){
|
| + score += pData->aIDF[i] * (
|
| + ( aFreq[i] * (k1 + 1.0) ) /
|
| + ( aFreq[i] + k1 * (1 - b + b * D / pData->avgdl) )
|
| + );
|
| + }
|
| +
|
| + /* If no error has occurred, return the calculated score. Otherwise,
|
| + ** throw an SQL exception. */
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_result_double(pCtx, -1.0 * score);
|
| + }else{
|
| + sqlite3_result_error_code(pCtx, rc);
|
| + }
|
| +}
|
| +
|
| +static int sqlite3Fts5AuxInit(fts5_api *pApi){
|
| + struct Builtin {
|
| + const char *zFunc; /* Function name (nul-terminated) */
|
| + void *pUserData; /* User-data pointer */
|
| + fts5_extension_function xFunc;/* Callback function */
|
| + void (*xDestroy)(void*); /* Destructor function */
|
| + } aBuiltin [] = {
|
| + { "snippet", 0, fts5SnippetFunction, 0 },
|
| + { "highlight", 0, fts5HighlightFunction, 0 },
|
| + { "bm25", 0, fts5Bm25Function, 0 },
|
| + };
|
| + int rc = SQLITE_OK; /* Return code */
|
| + int i; /* To iterate through builtin functions */
|
| +
|
| + for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){
|
| + rc = pApi->xCreateFunction(pApi,
|
| + aBuiltin[i].zFunc,
|
| + aBuiltin[i].pUserData,
|
| + aBuiltin[i].xFunc,
|
| + aBuiltin[i].xDestroy
|
| + );
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +
|
| +/*
|
| +** 2014 May 31
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +*/
|
| +
|
| +
|
| +
|
| +/* #include "fts5Int.h" */
|
| +
|
| +static int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, int nByte){
|
| + int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
|
| + u8 *pNew;
|
| + while( nNew<nByte ){
|
| + nNew = nNew * 2;
|
| + }
|
| + pNew = sqlite3_realloc(pBuf->p, nNew);
|
| + if( pNew==0 ){
|
| + *pRc = SQLITE_NOMEM;
|
| + return 1;
|
| + }else{
|
| + pBuf->nSpace = nNew;
|
| + pBuf->p = pNew;
|
| + }
|
| + return 0;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Encode value iVal as an SQLite varint and append it to the buffer object
|
| +** pBuf. If an OOM error occurs, set the error code in p.
|
| +*/
|
| +static void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
|
| + if( fts5BufferGrow(pRc, pBuf, 9) ) return;
|
| + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal);
|
| +}
|
| +
|
| +static void sqlite3Fts5Put32(u8 *aBuf, int iVal){
|
| + aBuf[0] = (iVal>>24) & 0x00FF;
|
| + aBuf[1] = (iVal>>16) & 0x00FF;
|
| + aBuf[2] = (iVal>> 8) & 0x00FF;
|
| + aBuf[3] = (iVal>> 0) & 0x00FF;
|
| +}
|
| +
|
| +static int sqlite3Fts5Get32(const u8 *aBuf){
|
| + return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3];
|
| +}
|
| +
|
| +/*
|
| +** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set
|
| +** the error code in p. If an error has already occurred when this function
|
| +** is called, it is a no-op.
|
| +*/
|
| +static void sqlite3Fts5BufferAppendBlob(
|
| + int *pRc,
|
| + Fts5Buffer *pBuf,
|
| + int nData,
|
| + const u8 *pData
|
| +){
|
| + assert( *pRc || nData>=0 );
|
| + if( fts5BufferGrow(pRc, pBuf, nData) ) return;
|
| + memcpy(&pBuf->p[pBuf->n], pData, nData);
|
| + pBuf->n += nData;
|
| +}
|
| +
|
| +/*
|
| +** Append the nul-terminated string zStr to the buffer pBuf. This function
|
| +** ensures that the byte following the buffer data is set to 0x00, even
|
| +** though this byte is not included in the pBuf->n count.
|
| +*/
|
| +static void sqlite3Fts5BufferAppendString(
|
| + int *pRc,
|
| + Fts5Buffer *pBuf,
|
| + const char *zStr
|
| +){
|
| + int nStr = (int)strlen(zStr);
|
| + sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr+1, (const u8*)zStr);
|
| + pBuf->n--;
|
| +}
|
| +
|
| +/*
|
| +** Argument zFmt is a printf() style format string. This function performs
|
| +** the printf() style processing, then appends the results to buffer pBuf.
|
| +**
|
| +** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte
|
| +** following the buffer data is set to 0x00, even though this byte is not
|
| +** included in the pBuf->n count.
|
| +*/
|
| +static void sqlite3Fts5BufferAppendPrintf(
|
| + int *pRc,
|
| + Fts5Buffer *pBuf,
|
| + char *zFmt, ...
|
| +){
|
| + if( *pRc==SQLITE_OK ){
|
| + char *zTmp;
|
| + va_list ap;
|
| + va_start(ap, zFmt);
|
| + zTmp = sqlite3_vmprintf(zFmt, ap);
|
| + va_end(ap);
|
| +
|
| + if( zTmp==0 ){
|
| + *pRc = SQLITE_NOMEM;
|
| + }else{
|
| + sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp);
|
| + sqlite3_free(zTmp);
|
| + }
|
| + }
|
| +}
|
| +
|
| +static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...){
|
| + char *zRet = 0;
|
| + if( *pRc==SQLITE_OK ){
|
| + va_list ap;
|
| + va_start(ap, zFmt);
|
| + zRet = sqlite3_vmprintf(zFmt, ap);
|
| + va_end(ap);
|
| + if( zRet==0 ){
|
| + *pRc = SQLITE_NOMEM;
|
| + }
|
| + }
|
| + return zRet;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Free any buffer allocated by pBuf. Zero the structure before returning.
|
| +*/
|
| +static void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){
|
| + sqlite3_free(pBuf->p);
|
| + memset(pBuf, 0, sizeof(Fts5Buffer));
|
| +}
|
| +
|
| +/*
|
| +** Zero the contents of the buffer object. But do not free the associated
|
| +** memory allocation.
|
| +*/
|
| +static void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){
|
| + pBuf->n = 0;
|
| +}
|
| +
|
| +/*
|
| +** Set the buffer to contain nData/pData. If an OOM error occurs, leave an
|
| +** the error code in p. If an error has already occurred when this function
|
| +** is called, it is a no-op.
|
| +*/
|
| +static void sqlite3Fts5BufferSet(
|
| + int *pRc,
|
| + Fts5Buffer *pBuf,
|
| + int nData,
|
| + const u8 *pData
|
| +){
|
| + pBuf->n = 0;
|
| + sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData);
|
| +}
|
| +
|
| +static int sqlite3Fts5PoslistNext64(
|
| + const u8 *a, int n, /* Buffer containing poslist */
|
| + int *pi, /* IN/OUT: Offset within a[] */
|
| + i64 *piOff /* IN/OUT: Current offset */
|
| +){
|
| + int i = *pi;
|
| + if( i>=n ){
|
| + /* EOF */
|
| + *piOff = -1;
|
| + return 1;
|
| + }else{
|
| + i64 iOff = *piOff;
|
| + int iVal;
|
| + fts5FastGetVarint32(a, i, iVal);
|
| + if( iVal==1 ){
|
| + fts5FastGetVarint32(a, i, iVal);
|
| + iOff = ((i64)iVal) << 32;
|
| + fts5FastGetVarint32(a, i, iVal);
|
| + }
|
| + *piOff = iOff + (iVal-2);
|
| + *pi = i;
|
| + return 0;
|
| + }
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Advance the iterator object passed as the only argument. Return true
|
| +** if the iterator reaches EOF, or false otherwise.
|
| +*/
|
| +static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){
|
| + if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) ){
|
| + pIter->bEof = 1;
|
| + }
|
| + return pIter->bEof;
|
| +}
|
| +
|
| +static int sqlite3Fts5PoslistReaderInit(
|
| + const u8 *a, int n, /* Poslist buffer to iterate through */
|
| + Fts5PoslistReader *pIter /* Iterator object to initialize */
|
| +){
|
| + memset(pIter, 0, sizeof(*pIter));
|
| + pIter->a = a;
|
| + pIter->n = n;
|
| + sqlite3Fts5PoslistReaderNext(pIter);
|
| + return pIter->bEof;
|
| +}
|
| +
|
| +static int sqlite3Fts5PoslistWriterAppend(
|
| + Fts5Buffer *pBuf,
|
| + Fts5PoslistWriter *pWriter,
|
| + i64 iPos
|
| +){
|
| + static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
|
| + int rc = SQLITE_OK;
|
| + if( 0==fts5BufferGrow(&rc, pBuf, 5+5+5) ){
|
| + if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
|
| + pBuf->p[pBuf->n++] = 1;
|
| + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
|
| + pWriter->iPrev = (iPos & colmask);
|
| + }
|
| + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
|
| + pWriter->iPrev = iPos;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static void *sqlite3Fts5MallocZero(int *pRc, int nByte){
|
| + void *pRet = 0;
|
| + if( *pRc==SQLITE_OK ){
|
| + pRet = sqlite3_malloc(nByte);
|
| + if( pRet==0 && nByte>0 ){
|
| + *pRc = SQLITE_NOMEM;
|
| + }else{
|
| + memset(pRet, 0, nByte);
|
| + }
|
| + }
|
| + return pRet;
|
| +}
|
| +
|
| +/*
|
| +** Return a nul-terminated copy of the string indicated by pIn. If nIn
|
| +** is non-negative, then it is the length of the string in bytes. Otherwise,
|
| +** the length of the string is determined using strlen().
|
| +**
|
| +** It is the responsibility of the caller to eventually free the returned
|
| +** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned.
|
| +*/
|
| +static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){
|
| + char *zRet = 0;
|
| + if( *pRc==SQLITE_OK ){
|
| + if( nIn<0 ){
|
| + nIn = (int)strlen(pIn);
|
| + }
|
| + zRet = (char*)sqlite3_malloc(nIn+1);
|
| + if( zRet ){
|
| + memcpy(zRet, pIn, nIn);
|
| + zRet[nIn] = '\0';
|
| + }else{
|
| + *pRc = SQLITE_NOMEM;
|
| + }
|
| + }
|
| + return zRet;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Return true if character 't' may be part of an FTS5 bareword, or false
|
| +** otherwise. Characters that may be part of barewords:
|
| +**
|
| +** * All non-ASCII characters,
|
| +** * The 52 upper and lower case ASCII characters, and
|
| +** * The 10 integer ASCII characters.
|
| +** * The underscore character "_" (0x5F).
|
| +** * The unicode "subsitute" character (0x1A).
|
| +*/
|
| +static int sqlite3Fts5IsBareword(char t){
|
| + u8 aBareword[128] = {
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */
|
| + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */
|
| + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */
|
| + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */
|
| + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */
|
| + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */
|
| + };
|
| +
|
| + return (t & 0x80) || aBareword[(int)t];
|
| +}
|
| +
|
| +
|
| +
|
| +/*
|
| +** 2014 Jun 09
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +** This is an SQLite module implementing full-text search.
|
| +*/
|
| +
|
| +
|
| +
|
| +/* #include "fts5Int.h" */
|
| +
|
| +#define FTS5_DEFAULT_PAGE_SIZE 4050
|
| +#define FTS5_DEFAULT_AUTOMERGE 4
|
| +#define FTS5_DEFAULT_CRISISMERGE 16
|
| +#define FTS5_DEFAULT_HASHSIZE (1024*1024)
|
| +
|
| +/* Maximum allowed page size */
|
| +#define FTS5_MAX_PAGE_SIZE (128*1024)
|
| +
|
| +static int fts5_iswhitespace(char x){
|
| + return (x==' ');
|
| +}
|
| +
|
| +static int fts5_isopenquote(char x){
|
| + return (x=='"' || x=='\'' || x=='[' || x=='`');
|
| +}
|
| +
|
| +/*
|
| +** Argument pIn points to a character that is part of a nul-terminated
|
| +** string. Return a pointer to the first character following *pIn in
|
| +** the string that is not a white-space character.
|
| +*/
|
| +static const char *fts5ConfigSkipWhitespace(const char *pIn){
|
| + const char *p = pIn;
|
| + if( p ){
|
| + while( fts5_iswhitespace(*p) ){ p++; }
|
| + }
|
| + return p;
|
| +}
|
| +
|
| +/*
|
| +** Argument pIn points to a character that is part of a nul-terminated
|
| +** string. Return a pointer to the first character following *pIn in
|
| +** the string that is not a "bareword" character.
|
| +*/
|
| +static const char *fts5ConfigSkipBareword(const char *pIn){
|
| + const char *p = pIn;
|
| + while ( sqlite3Fts5IsBareword(*p) ) p++;
|
| + if( p==pIn ) p = 0;
|
| + return p;
|
| +}
|
| +
|
| +static int fts5_isdigit(char a){
|
| + return (a>='0' && a<='9');
|
| +}
|
| +
|
| +
|
| +
|
| +static const char *fts5ConfigSkipLiteral(const char *pIn){
|
| + const char *p = pIn;
|
| + switch( *p ){
|
| + case 'n': case 'N':
|
| + if( sqlite3_strnicmp("null", p, 4)==0 ){
|
| + p = &p[4];
|
| + }else{
|
| + p = 0;
|
| + }
|
| + break;
|
| +
|
| + case 'x': case 'X':
|
| + p++;
|
| + if( *p=='\'' ){
|
| + p++;
|
| + while( (*p>='a' && *p<='f')
|
| + || (*p>='A' && *p<='F')
|
| + || (*p>='0' && *p<='9')
|
| + ){
|
| + p++;
|
| + }
|
| + if( *p=='\'' && 0==((p-pIn)%2) ){
|
| + p++;
|
| + }else{
|
| + p = 0;
|
| + }
|
| + }else{
|
| + p = 0;
|
| + }
|
| + break;
|
| +
|
| + case '\'':
|
| + p++;
|
| + while( p ){
|
| + if( *p=='\'' ){
|
| + p++;
|
| + if( *p!='\'' ) break;
|
| + }
|
| + p++;
|
| + if( *p==0 ) p = 0;
|
| + }
|
| + break;
|
| +
|
| + default:
|
| + /* maybe a number */
|
| + if( *p=='+' || *p=='-' ) p++;
|
| + while( fts5_isdigit(*p) ) p++;
|
| +
|
| + /* At this point, if the literal was an integer, the parse is
|
| + ** finished. Or, if it is a floating point value, it may continue
|
| + ** with either a decimal point or an 'E' character. */
|
| + if( *p=='.' && fts5_isdigit(p[1]) ){
|
| + p += 2;
|
| + while( fts5_isdigit(*p) ) p++;
|
| + }
|
| + if( p==pIn ) p = 0;
|
| +
|
| + break;
|
| + }
|
| +
|
| + return p;
|
| +}
|
| +
|
| +/*
|
| +** The first character of the string pointed to by argument z is guaranteed
|
| +** to be an open-quote character (see function fts5_isopenquote()).
|
| +**
|
| +** This function searches for the corresponding close-quote character within
|
| +** the string and, if found, dequotes the string in place and adds a new
|
| +** nul-terminator byte.
|
| +**
|
| +** If the close-quote is found, the value returned is the byte offset of
|
| +** the character immediately following it. Or, if the close-quote is not
|
| +** found, -1 is returned. If -1 is returned, the buffer is left in an
|
| +** undefined state.
|
| +*/
|
| +static int fts5Dequote(char *z){
|
| + char q;
|
| + int iIn = 1;
|
| + int iOut = 0;
|
| + q = z[0];
|
| +
|
| + /* Set stack variable q to the close-quote character */
|
| + assert( q=='[' || q=='\'' || q=='"' || q=='`' );
|
| + if( q=='[' ) q = ']';
|
| +
|
| + while( ALWAYS(z[iIn]) ){
|
| + if( z[iIn]==q ){
|
| + if( z[iIn+1]!=q ){
|
| + /* Character iIn was the close quote. */
|
| + iIn++;
|
| + break;
|
| + }else{
|
| + /* Character iIn and iIn+1 form an escaped quote character. Skip
|
| + ** the input cursor past both and copy a single quote character
|
| + ** to the output buffer. */
|
| + iIn += 2;
|
| + z[iOut++] = q;
|
| + }
|
| + }else{
|
| + z[iOut++] = z[iIn++];
|
| + }
|
| + }
|
| +
|
| + z[iOut] = '\0';
|
| + return iIn;
|
| +}
|
| +
|
| +/*
|
| +** Convert an SQL-style quoted string into a normal string by removing
|
| +** the quote characters. The conversion is done in-place. If the
|
| +** input does not begin with a quote character, then this routine
|
| +** is a no-op.
|
| +**
|
| +** Examples:
|
| +**
|
| +** "abc" becomes abc
|
| +** 'xyz' becomes xyz
|
| +** [pqr] becomes pqr
|
| +** `mno` becomes mno
|
| +*/
|
| +static void sqlite3Fts5Dequote(char *z){
|
| + char quote; /* Quote character (if any ) */
|
| +
|
| + assert( 0==fts5_iswhitespace(z[0]) );
|
| + quote = z[0];
|
| + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
|
| + fts5Dequote(z);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Parse a "special" CREATE VIRTUAL TABLE directive and update
|
| +** configuration object pConfig as appropriate.
|
| +**
|
| +** If successful, object pConfig is updated and SQLITE_OK returned. If
|
| +** an error occurs, an SQLite error code is returned and an error message
|
| +** may be left in *pzErr. It is the responsibility of the caller to
|
| +** eventually free any such error message using sqlite3_free().
|
| +*/
|
| +static int fts5ConfigParseSpecial(
|
| + Fts5Global *pGlobal,
|
| + Fts5Config *pConfig, /* Configuration object to update */
|
| + const char *zCmd, /* Special command to parse */
|
| + const char *zArg, /* Argument to parse */
|
| + char **pzErr /* OUT: Error message */
|
| +){
|
| + int rc = SQLITE_OK;
|
| + int nCmd = (int)strlen(zCmd);
|
| + if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
|
| + const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
|
| + const char *p;
|
| + int bFirst = 1;
|
| + if( pConfig->aPrefix==0 ){
|
| + pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
|
| + if( rc ) return rc;
|
| + }
|
| +
|
| + p = zArg;
|
| + while( 1 ){
|
| + int nPre = 0;
|
| +
|
| + while( p[0]==' ' ) p++;
|
| + if( bFirst==0 && p[0]==',' ){
|
| + p++;
|
| + while( p[0]==' ' ) p++;
|
| + }else if( p[0]=='\0' ){
|
| + break;
|
| + }
|
| + if( p[0]<'0' || p[0]>'9' ){
|
| + *pzErr = sqlite3_mprintf("malformed prefix=... directive");
|
| + rc = SQLITE_ERROR;
|
| + break;
|
| + }
|
| +
|
| + if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES ){
|
| + *pzErr = sqlite3_mprintf(
|
| + "too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES
|
| + );
|
| + rc = SQLITE_ERROR;
|
| + break;
|
| + }
|
| +
|
| + while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
|
| + nPre = nPre*10 + (p[0] - '0');
|
| + p++;
|
| + }
|
| +
|
| + if( rc==SQLITE_OK && (nPre<=0 || nPre>=1000) ){
|
| + *pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
|
| + rc = SQLITE_ERROR;
|
| + break;
|
| + }
|
| +
|
| + pConfig->aPrefix[pConfig->nPrefix] = nPre;
|
| + pConfig->nPrefix++;
|
| + bFirst = 0;
|
| + }
|
| + assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES );
|
| + return rc;
|
| + }
|
| +
|
| + if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
|
| + const char *p = (const char*)zArg;
|
| + int nArg = (int)strlen(zArg) + 1;
|
| + char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
|
| + char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
|
| + char *pSpace = pDel;
|
| +
|
| + if( azArg && pSpace ){
|
| + if( pConfig->pTok ){
|
| + *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + for(nArg=0; p && *p; nArg++){
|
| + const char *p2 = fts5ConfigSkipWhitespace(p);
|
| + if( *p2=='\'' ){
|
| + p = fts5ConfigSkipLiteral(p2);
|
| + }else{
|
| + p = fts5ConfigSkipBareword(p2);
|
| + }
|
| + if( p ){
|
| + memcpy(pSpace, p2, p-p2);
|
| + azArg[nArg] = pSpace;
|
| + sqlite3Fts5Dequote(pSpace);
|
| + pSpace += (p - p2) + 1;
|
| + p = fts5ConfigSkipWhitespace(p);
|
| + }
|
| + }
|
| + if( p==0 ){
|
| + *pzErr = sqlite3_mprintf("parse error in tokenize directive");
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + rc = sqlite3Fts5GetTokenizer(pGlobal,
|
| + (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi,
|
| + pzErr
|
| + );
|
| + }
|
| + }
|
| + }
|
| +
|
| + sqlite3_free(azArg);
|
| + sqlite3_free(pDel);
|
| + return rc;
|
| + }
|
| +
|
| + if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
|
| + if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
|
| + *pzErr = sqlite3_mprintf("multiple content=... directives");
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + if( zArg[0] ){
|
| + pConfig->eContent = FTS5_CONTENT_EXTERNAL;
|
| + pConfig->zContent = sqlite3Fts5Mprintf(&rc, "%Q.%Q", pConfig->zDb,zArg);
|
| + }else{
|
| + pConfig->eContent = FTS5_CONTENT_NONE;
|
| + }
|
| + }
|
| + return rc;
|
| + }
|
| +
|
| + if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
|
| + if( pConfig->zContentRowid ){
|
| + *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1);
|
| + }
|
| + return rc;
|
| + }
|
| +
|
| + if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){
|
| + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
|
| + *pzErr = sqlite3_mprintf("malformed columnsize=... directive");
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + pConfig->bColumnsize = (zArg[0]=='1');
|
| + }
|
| + return rc;
|
| + }
|
| +
|
| + *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
|
| + return SQLITE_ERROR;
|
| +}
|
| +
|
| +/*
|
| +** Allocate an instance of the default tokenizer ("simple") at
|
| +** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
|
| +** code if an error occurs.
|
| +*/
|
| +static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
|
| + assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
|
| + return sqlite3Fts5GetTokenizer(
|
| + pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0
|
| + );
|
| +}
|
| +
|
| +/*
|
| +** Gobble up the first bareword or quoted word from the input buffer zIn.
|
| +** Return a pointer to the character immediately following the last in
|
| +** the gobbled word if successful, or a NULL pointer otherwise (failed
|
| +** to find close-quote character).
|
| +**
|
| +** Before returning, set pzOut to point to a new buffer containing a
|
| +** nul-terminated, dequoted copy of the gobbled word. If the word was
|
| +** quoted, *pbQuoted is also set to 1 before returning.
|
| +**
|
| +** If *pRc is other than SQLITE_OK when this function is called, it is
|
| +** a no-op (NULL is returned). Otherwise, if an OOM occurs within this
|
| +** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not*
|
| +** set if a parse error (failed to find close quote) occurs.
|
| +*/
|
| +static const char *fts5ConfigGobbleWord(
|
| + int *pRc, /* IN/OUT: Error code */
|
| + const char *zIn, /* Buffer to gobble string/bareword from */
|
| + char **pzOut, /* OUT: malloc'd buffer containing str/bw */
|
| + int *pbQuoted /* OUT: Set to true if dequoting required */
|
| +){
|
| + const char *zRet = 0;
|
| +
|
| + int nIn = (int)strlen(zIn);
|
| + char *zOut = sqlite3_malloc(nIn+1);
|
| +
|
| + assert( *pRc==SQLITE_OK );
|
| + *pbQuoted = 0;
|
| + *pzOut = 0;
|
| +
|
| + if( zOut==0 ){
|
| + *pRc = SQLITE_NOMEM;
|
| + }else{
|
| + memcpy(zOut, zIn, nIn+1);
|
| + if( fts5_isopenquote(zOut[0]) ){
|
| + int ii = fts5Dequote(zOut);
|
| + zRet = &zIn[ii];
|
| + *pbQuoted = 1;
|
| + }else{
|
| + zRet = fts5ConfigSkipBareword(zIn);
|
| + zOut[zRet-zIn] = '\0';
|
| + }
|
| + }
|
| +
|
| + if( zRet==0 ){
|
| + sqlite3_free(zOut);
|
| + }else{
|
| + *pzOut = zOut;
|
| + }
|
| +
|
| + return zRet;
|
| +}
|
| +
|
| +static int fts5ConfigParseColumn(
|
| + Fts5Config *p,
|
| + char *zCol,
|
| + char *zArg,
|
| + char **pzErr
|
| +){
|
| + int rc = SQLITE_OK;
|
| + if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME)
|
| + || 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME)
|
| + ){
|
| + *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol);
|
| + rc = SQLITE_ERROR;
|
| + }else if( zArg ){
|
| + if( 0==sqlite3_stricmp(zArg, "unindexed") ){
|
| + p->abUnindexed[p->nCol] = 1;
|
| + }else{
|
| + *pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg);
|
| + rc = SQLITE_ERROR;
|
| + }
|
| + }
|
| +
|
| + p->azCol[p->nCol++] = zCol;
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Populate the Fts5Config.zContentExprlist string.
|
| +*/
|
| +static int fts5ConfigMakeExprlist(Fts5Config *p){
|
| + int i;
|
| + int rc = SQLITE_OK;
|
| + Fts5Buffer buf = {0, 0, 0};
|
| +
|
| + sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid);
|
| + if( p->eContent!=FTS5_CONTENT_NONE ){
|
| + for(i=0; i<p->nCol; i++){
|
| + if( p->eContent==FTS5_CONTENT_EXTERNAL ){
|
| + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]);
|
| + }else{
|
| + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i);
|
| + }
|
| + }
|
| + }
|
| +
|
| + assert( p->zContentExprlist==0 );
|
| + p->zContentExprlist = (char*)buf.p;
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Arguments nArg/azArg contain the string arguments passed to the xCreate
|
| +** or xConnect method of the virtual table. This function attempts to
|
| +** allocate an instance of Fts5Config containing the results of parsing
|
| +** those arguments.
|
| +**
|
| +** If successful, SQLITE_OK is returned and *ppOut is set to point to the
|
| +** new Fts5Config object. If an error occurs, an SQLite error code is
|
| +** returned, *ppOut is set to NULL and an error message may be left in
|
| +** *pzErr. It is the responsibility of the caller to eventually free any
|
| +** such error message using sqlite3_free().
|
| +*/
|
| +static int sqlite3Fts5ConfigParse(
|
| + Fts5Global *pGlobal,
|
| + sqlite3 *db,
|
| + int nArg, /* Number of arguments */
|
| + const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */
|
| + Fts5Config **ppOut, /* OUT: Results of parse */
|
| + char **pzErr /* OUT: Error message */
|
| +){
|
| + int rc = SQLITE_OK; /* Return code */
|
| + Fts5Config *pRet; /* New object to return */
|
| + int i;
|
| + int nByte;
|
| +
|
| + *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
|
| + if( pRet==0 ) return SQLITE_NOMEM;
|
| + memset(pRet, 0, sizeof(Fts5Config));
|
| + pRet->db = db;
|
| + pRet->iCookie = -1;
|
| +
|
| + nByte = nArg * (sizeof(char*) + sizeof(u8));
|
| + pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
|
| + pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
|
| + pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
|
| + pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
|
| + pRet->bColumnsize = 1;
|
| +#ifdef SQLITE_DEBUG
|
| + pRet->bPrefixIndex = 1;
|
| +#endif
|
| + if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
|
| + *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
|
| + rc = SQLITE_ERROR;
|
| + }
|
| +
|
| + for(i=3; rc==SQLITE_OK && i<nArg; i++){
|
| + const char *zOrig = azArg[i];
|
| + const char *z;
|
| + char *zOne = 0;
|
| + char *zTwo = 0;
|
| + int bOption = 0;
|
| + int bMustBeCol = 0;
|
| +
|
| + z = fts5ConfigGobbleWord(&rc, zOrig, &zOne, &bMustBeCol);
|
| + z = fts5ConfigSkipWhitespace(z);
|
| + if( z && *z=='=' ){
|
| + bOption = 1;
|
| + z++;
|
| + if( bMustBeCol ) z = 0;
|
| + }
|
| + z = fts5ConfigSkipWhitespace(z);
|
| + if( z && z[0] ){
|
| + int bDummy;
|
| + z = fts5ConfigGobbleWord(&rc, z, &zTwo, &bDummy);
|
| + if( z && z[0] ) z = 0;
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + if( z==0 ){
|
| + *pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig);
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + if( bOption ){
|
| + rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr);
|
| + }else{
|
| + rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
|
| + zOne = 0;
|
| + }
|
| + }
|
| + }
|
| +
|
| + sqlite3_free(zOne);
|
| + sqlite3_free(zTwo);
|
| + }
|
| +
|
| + /* If a tokenizer= option was successfully parsed, the tokenizer has
|
| + ** already been allocated. Otherwise, allocate an instance of the default
|
| + ** tokenizer (unicode61) now. */
|
| + if( rc==SQLITE_OK && pRet->pTok==0 ){
|
| + rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
|
| + }
|
| +
|
| + /* If no zContent option was specified, fill in the default values. */
|
| + if( rc==SQLITE_OK && pRet->zContent==0 ){
|
| + const char *zTail = 0;
|
| + assert( pRet->eContent==FTS5_CONTENT_NORMAL
|
| + || pRet->eContent==FTS5_CONTENT_NONE
|
| + );
|
| + if( pRet->eContent==FTS5_CONTENT_NORMAL ){
|
| + zTail = "content";
|
| + }else if( pRet->bColumnsize ){
|
| + zTail = "docsize";
|
| + }
|
| +
|
| + if( zTail ){
|
| + pRet->zContent = sqlite3Fts5Mprintf(
|
| + &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail
|
| + );
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK && pRet->zContentRowid==0 ){
|
| + pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1);
|
| + }
|
| +
|
| + /* Formulate the zContentExprlist text */
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5ConfigMakeExprlist(pRet);
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + sqlite3Fts5ConfigFree(pRet);
|
| + *ppOut = 0;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Free the configuration object passed as the only argument.
|
| +*/
|
| +static void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
|
| + if( pConfig ){
|
| + int i;
|
| + if( pConfig->pTok ){
|
| + pConfig->pTokApi->xDelete(pConfig->pTok);
|
| + }
|
| + sqlite3_free(pConfig->zDb);
|
| + sqlite3_free(pConfig->zName);
|
| + for(i=0; i<pConfig->nCol; i++){
|
| + sqlite3_free(pConfig->azCol[i]);
|
| + }
|
| + sqlite3_free(pConfig->azCol);
|
| + sqlite3_free(pConfig->aPrefix);
|
| + sqlite3_free(pConfig->zRank);
|
| + sqlite3_free(pConfig->zRankArgs);
|
| + sqlite3_free(pConfig->zContent);
|
| + sqlite3_free(pConfig->zContentRowid);
|
| + sqlite3_free(pConfig->zContentExprlist);
|
| + sqlite3_free(pConfig);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Call sqlite3_declare_vtab() based on the contents of the configuration
|
| +** object passed as the only argument. Return SQLITE_OK if successful, or
|
| +** an SQLite error code if an error occurs.
|
| +*/
|
| +static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){
|
| + int i;
|
| + int rc = SQLITE_OK;
|
| + char *zSql;
|
| +
|
| + zSql = sqlite3Fts5Mprintf(&rc, "CREATE TABLE x(");
|
| + for(i=0; zSql && i<pConfig->nCol; i++){
|
| + const char *zSep = (i==0?"":", ");
|
| + zSql = sqlite3Fts5Mprintf(&rc, "%z%s%Q", zSql, zSep, pConfig->azCol[i]);
|
| + }
|
| + zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)",
|
| + zSql, pConfig->zName, FTS5_RANK_NAME
|
| + );
|
| +
|
| + assert( zSql || rc==SQLITE_NOMEM );
|
| + if( zSql ){
|
| + rc = sqlite3_declare_vtab(pConfig->db, zSql);
|
| + sqlite3_free(zSql);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Tokenize the text passed via the second and third arguments.
|
| +**
|
| +** The callback is invoked once for each token in the input text. The
|
| +** arguments passed to it are, in order:
|
| +**
|
| +** void *pCtx // Copy of 4th argument to sqlite3Fts5Tokenize()
|
| +** const char *pToken // Pointer to buffer containing token
|
| +** int nToken // Size of token in bytes
|
| +** int iStart // Byte offset of start of token within input text
|
| +** int iEnd // Byte offset of end of token within input text
|
| +** int iPos // Position of token in input (first token is 0)
|
| +**
|
| +** If the callback returns a non-zero value the tokenization is abandoned
|
| +** and no further callbacks are issued.
|
| +**
|
| +** This function returns SQLITE_OK if successful or an SQLite error code
|
| +** if an error occurs. If the tokenization was abandoned early because
|
| +** the callback returned SQLITE_DONE, this is not an error and this function
|
| +** still returns SQLITE_OK. Or, if the tokenization was abandoned early
|
| +** because the callback returned another non-zero value, it is assumed
|
| +** to be an SQLite error code and returned to the caller.
|
| +*/
|
| +static int sqlite3Fts5Tokenize(
|
| + Fts5Config *pConfig, /* FTS5 Configuration object */
|
| + int flags, /* FTS5_TOKENIZE_* flags */
|
| + const char *pText, int nText, /* Text to tokenize */
|
| + void *pCtx, /* Context passed to xToken() */
|
| + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
|
| +){
|
| + if( pText==0 ) return SQLITE_OK;
|
| + return pConfig->pTokApi->xTokenize(
|
| + pConfig->pTok, pCtx, flags, pText, nText, xToken
|
| + );
|
| +}
|
| +
|
| +/*
|
| +** Argument pIn points to the first character in what is expected to be
|
| +** a comma-separated list of SQL literals followed by a ')' character.
|
| +** If it actually is this, return a pointer to the ')'. Otherwise, return
|
| +** NULL to indicate a parse error.
|
| +*/
|
| +static const char *fts5ConfigSkipArgs(const char *pIn){
|
| + const char *p = pIn;
|
| +
|
| + while( 1 ){
|
| + p = fts5ConfigSkipWhitespace(p);
|
| + p = fts5ConfigSkipLiteral(p);
|
| + p = fts5ConfigSkipWhitespace(p);
|
| + if( p==0 || *p==')' ) break;
|
| + if( *p!=',' ){
|
| + p = 0;
|
| + break;
|
| + }
|
| + p++;
|
| + }
|
| +
|
| + return p;
|
| +}
|
| +
|
| +/*
|
| +** Parameter zIn contains a rank() function specification. The format of
|
| +** this is:
|
| +**
|
| +** + Bareword (function name)
|
| +** + Open parenthesis - "("
|
| +** + Zero or more SQL literals in a comma separated list
|
| +** + Close parenthesis - ")"
|
| +*/
|
| +static int sqlite3Fts5ConfigParseRank(
|
| + const char *zIn, /* Input string */
|
| + char **pzRank, /* OUT: Rank function name */
|
| + char **pzRankArgs /* OUT: Rank function arguments */
|
| +){
|
| + const char *p = zIn;
|
| + const char *pRank;
|
| + char *zRank = 0;
|
| + char *zRankArgs = 0;
|
| + int rc = SQLITE_OK;
|
| +
|
| + *pzRank = 0;
|
| + *pzRankArgs = 0;
|
| +
|
| + if( p==0 ){
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + p = fts5ConfigSkipWhitespace(p);
|
| + pRank = p;
|
| + p = fts5ConfigSkipBareword(p);
|
| +
|
| + if( p ){
|
| + zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
|
| + if( zRank ) memcpy(zRank, pRank, p-pRank);
|
| + }else{
|
| + rc = SQLITE_ERROR;
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + p = fts5ConfigSkipWhitespace(p);
|
| + if( *p!='(' ) rc = SQLITE_ERROR;
|
| + p++;
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + const char *pArgs;
|
| + p = fts5ConfigSkipWhitespace(p);
|
| + pArgs = p;
|
| + if( *p!=')' ){
|
| + p = fts5ConfigSkipArgs(p);
|
| + if( p==0 ){
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
|
| + if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + sqlite3_free(zRank);
|
| + assert( zRankArgs==0 );
|
| + }else{
|
| + *pzRank = zRank;
|
| + *pzRankArgs = zRankArgs;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static int sqlite3Fts5ConfigSetValue(
|
| + Fts5Config *pConfig,
|
| + const char *zKey,
|
| + sqlite3_value *pVal,
|
| + int *pbBadkey
|
| +){
|
| + int rc = SQLITE_OK;
|
| +
|
| + if( 0==sqlite3_stricmp(zKey, "pgsz") ){
|
| + int pgsz = 0;
|
| + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
|
| + pgsz = sqlite3_value_int(pVal);
|
| + }
|
| + if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){
|
| + *pbBadkey = 1;
|
| + }else{
|
| + pConfig->pgsz = pgsz;
|
| + }
|
| + }
|
| +
|
| + else if( 0==sqlite3_stricmp(zKey, "hashsize") ){
|
| + int nHashSize = -1;
|
| + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
|
| + nHashSize = sqlite3_value_int(pVal);
|
| + }
|
| + if( nHashSize<=0 ){
|
| + *pbBadkey = 1;
|
| + }else{
|
| + pConfig->nHashSize = nHashSize;
|
| + }
|
| + }
|
| +
|
| + else if( 0==sqlite3_stricmp(zKey, "automerge") ){
|
| + int nAutomerge = -1;
|
| + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
|
| + nAutomerge = sqlite3_value_int(pVal);
|
| + }
|
| + if( nAutomerge<0 || nAutomerge>64 ){
|
| + *pbBadkey = 1;
|
| + }else{
|
| + if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE;
|
| + pConfig->nAutomerge = nAutomerge;
|
| + }
|
| + }
|
| +
|
| + else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){
|
| + int nCrisisMerge = -1;
|
| + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
|
| + nCrisisMerge = sqlite3_value_int(pVal);
|
| + }
|
| + if( nCrisisMerge<0 ){
|
| + *pbBadkey = 1;
|
| + }else{
|
| + if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
|
| + pConfig->nCrisisMerge = nCrisisMerge;
|
| + }
|
| + }
|
| +
|
| + else if( 0==sqlite3_stricmp(zKey, "rank") ){
|
| + const char *zIn = (const char*)sqlite3_value_text(pVal);
|
| + char *zRank;
|
| + char *zRankArgs;
|
| + rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs);
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_free(pConfig->zRank);
|
| + sqlite3_free(pConfig->zRankArgs);
|
| + pConfig->zRank = zRank;
|
| + pConfig->zRankArgs = zRankArgs;
|
| + }else if( rc==SQLITE_ERROR ){
|
| + rc = SQLITE_OK;
|
| + *pbBadkey = 1;
|
| + }
|
| + }else{
|
| + *pbBadkey = 1;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Load the contents of the %_config table into memory.
|
| +*/
|
| +static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
|
| + const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
|
| + char *zSql;
|
| + sqlite3_stmt *p = 0;
|
| + int rc = SQLITE_OK;
|
| + int iVersion = 0;
|
| +
|
| + /* Set default values */
|
| + pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
|
| + pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
|
| + pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
|
| + pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
|
| +
|
| + zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
|
| + if( zSql ){
|
| + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
|
| + sqlite3_free(zSql);
|
| + }
|
| +
|
| + assert( rc==SQLITE_OK || p==0 );
|
| + if( rc==SQLITE_OK ){
|
| + while( SQLITE_ROW==sqlite3_step(p) ){
|
| + const char *zK = (const char*)sqlite3_column_text(p, 0);
|
| + sqlite3_value *pVal = sqlite3_column_value(p, 1);
|
| + if( 0==sqlite3_stricmp(zK, "version") ){
|
| + iVersion = sqlite3_value_int(pVal);
|
| + }else{
|
| + int bDummy = 0;
|
| + sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, &bDummy);
|
| + }
|
| + }
|
| + rc = sqlite3_finalize(p);
|
| + }
|
| +
|
| + if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){
|
| + rc = SQLITE_ERROR;
|
| + if( pConfig->pzErrmsg ){
|
| + assert( 0==*pConfig->pzErrmsg );
|
| + *pConfig->pzErrmsg = sqlite3_mprintf(
|
| + "invalid fts5 file format (found %d, expected %d) - run 'rebuild'",
|
| + iVersion, FTS5_CURRENT_VERSION
|
| + );
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + pConfig->iCookie = iCookie;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** 2014 May 31
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +*/
|
| +
|
| +
|
| +
|
| +/* #include "fts5Int.h" */
|
| +/* #include "fts5parse.h" */
|
| +
|
| +/*
|
| +** All token types in the generated fts5parse.h file are greater than 0.
|
| +*/
|
| +#define FTS5_EOF 0
|
| +
|
| +#define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
|
| +
|
| +typedef struct Fts5ExprTerm Fts5ExprTerm;
|
| +
|
| +/*
|
| +** Functions generated by lemon from fts5parse.y.
|
| +*/
|
| +static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(u64));
|
| +static void sqlite3Fts5ParserFree(void*, void (*freeProc)(void*));
|
| +static void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);
|
| +#ifndef NDEBUG
|
| +/* #include <stdio.h> */
|
| +static void sqlite3Fts5ParserTrace(FILE*, char*);
|
| +#endif
|
| +
|
| +
|
| +struct Fts5Expr {
|
| + Fts5Index *pIndex;
|
| + Fts5ExprNode *pRoot;
|
| + int bDesc; /* Iterate in descending rowid order */
|
| + int nPhrase; /* Number of phrases in expression */
|
| + Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */
|
| +};
|
| +
|
| +/*
|
| +** eType:
|
| +** Expression node type. Always one of:
|
| +**
|
| +** FTS5_AND (nChild, apChild valid)
|
| +** FTS5_OR (nChild, apChild valid)
|
| +** FTS5_NOT (nChild, apChild valid)
|
| +** FTS5_STRING (pNear valid)
|
| +** FTS5_TERM (pNear valid)
|
| +*/
|
| +struct Fts5ExprNode {
|
| + int eType; /* Node type */
|
| + int bEof; /* True at EOF */
|
| + int bNomatch; /* True if entry is not a match */
|
| +
|
| + i64 iRowid; /* Current rowid */
|
| + Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */
|
| +
|
| + /* Child nodes. For a NOT node, this array always contains 2 entries. For
|
| + ** AND or OR nodes, it contains 2 or more entries. */
|
| + int nChild; /* Number of child nodes */
|
| + Fts5ExprNode *apChild[1]; /* Array of child nodes */
|
| +};
|
| +
|
| +#define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING)
|
| +
|
| +/*
|
| +** An instance of the following structure represents a single search term
|
| +** or term prefix.
|
| +*/
|
| +struct Fts5ExprTerm {
|
| + int bPrefix; /* True for a prefix term */
|
| + char *zTerm; /* nul-terminated term */
|
| + Fts5IndexIter *pIter; /* Iterator for this term */
|
| + Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */
|
| +};
|
| +
|
| +/*
|
| +** A phrase. One or more terms that must appear in a contiguous sequence
|
| +** within a document for it to match.
|
| +*/
|
| +struct Fts5ExprPhrase {
|
| + Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */
|
| + Fts5Buffer poslist; /* Current position list */
|
| + int nTerm; /* Number of entries in aTerm[] */
|
| + Fts5ExprTerm aTerm[1]; /* Terms that make up this phrase */
|
| +};
|
| +
|
| +/*
|
| +** One or more phrases that must appear within a certain token distance of
|
| +** each other within each matching document.
|
| +*/
|
| +struct Fts5ExprNearset {
|
| + int nNear; /* NEAR parameter */
|
| + Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */
|
| + int nPhrase; /* Number of entries in aPhrase[] array */
|
| + Fts5ExprPhrase *apPhrase[1]; /* Array of phrase pointers */
|
| +};
|
| +
|
| +
|
| +/*
|
| +** Parse context.
|
| +*/
|
| +struct Fts5Parse {
|
| + Fts5Config *pConfig;
|
| + char *zErr;
|
| + int rc;
|
| + int nPhrase; /* Size of apPhrase array */
|
| + Fts5ExprPhrase **apPhrase; /* Array of all phrases */
|
| + Fts5ExprNode *pExpr; /* Result of a successful parse */
|
| +};
|
| +
|
| +static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
|
| + va_list ap;
|
| + va_start(ap, zFmt);
|
| + if( pParse->rc==SQLITE_OK ){
|
| + pParse->zErr = sqlite3_vmprintf(zFmt, ap);
|
| + pParse->rc = SQLITE_ERROR;
|
| + }
|
| + va_end(ap);
|
| +}
|
| +
|
| +static int fts5ExprIsspace(char t){
|
| + return t==' ' || t=='\t' || t=='\n' || t=='\r';
|
| +}
|
| +
|
| +/*
|
| +** Read the first token from the nul-terminated string at *pz.
|
| +*/
|
| +static int fts5ExprGetToken(
|
| + Fts5Parse *pParse,
|
| + const char **pz, /* IN/OUT: Pointer into buffer */
|
| + Fts5Token *pToken
|
| +){
|
| + const char *z = *pz;
|
| + int tok;
|
| +
|
| + /* Skip past any whitespace */
|
| + while( fts5ExprIsspace(*z) ) z++;
|
| +
|
| + pToken->p = z;
|
| + pToken->n = 1;
|
| + switch( *z ){
|
| + case '(': tok = FTS5_LP; break;
|
| + case ')': tok = FTS5_RP; break;
|
| + case '{': tok = FTS5_LCP; break;
|
| + case '}': tok = FTS5_RCP; break;
|
| + case ':': tok = FTS5_COLON; break;
|
| + case ',': tok = FTS5_COMMA; break;
|
| + case '+': tok = FTS5_PLUS; break;
|
| + case '*': tok = FTS5_STAR; break;
|
| + case '\0': tok = FTS5_EOF; break;
|
| +
|
| + case '"': {
|
| + const char *z2;
|
| + tok = FTS5_STRING;
|
| +
|
| + for(z2=&z[1]; 1; z2++){
|
| + if( z2[0]=='"' ){
|
| + z2++;
|
| + if( z2[0]!='"' ) break;
|
| + }
|
| + if( z2[0]=='\0' ){
|
| + sqlite3Fts5ParseError(pParse, "unterminated string");
|
| + return FTS5_EOF;
|
| + }
|
| + }
|
| + pToken->n = (z2 - z);
|
| + break;
|
| + }
|
| +
|
| + default: {
|
| + const char *z2;
|
| + if( sqlite3Fts5IsBareword(z[0])==0 ){
|
| + sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z);
|
| + return FTS5_EOF;
|
| + }
|
| + tok = FTS5_STRING;
|
| + for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++);
|
| + pToken->n = (z2 - z);
|
| + if( pToken->n==2 && memcmp(pToken->p, "OR", 2)==0 ) tok = FTS5_OR;
|
| + if( pToken->n==3 && memcmp(pToken->p, "NOT", 3)==0 ) tok = FTS5_NOT;
|
| + if( pToken->n==3 && memcmp(pToken->p, "AND", 3)==0 ) tok = FTS5_AND;
|
| + break;
|
| + }
|
| + }
|
| +
|
| + *pz = &pToken->p[pToken->n];
|
| + return tok;
|
| +}
|
| +
|
| +static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc((int)t); }
|
| +static void fts5ParseFree(void *p){ sqlite3_free(p); }
|
| +
|
| +static int sqlite3Fts5ExprNew(
|
| + Fts5Config *pConfig, /* FTS5 Configuration */
|
| + const char *zExpr, /* Expression text */
|
| + Fts5Expr **ppNew,
|
| + char **pzErr
|
| +){
|
| + Fts5Parse sParse;
|
| + Fts5Token token;
|
| + const char *z = zExpr;
|
| + int t; /* Next token type */
|
| + void *pEngine;
|
| + Fts5Expr *pNew;
|
| +
|
| + *ppNew = 0;
|
| + *pzErr = 0;
|
| + memset(&sParse, 0, sizeof(sParse));
|
| + pEngine = sqlite3Fts5ParserAlloc(fts5ParseAlloc);
|
| + if( pEngine==0 ){ return SQLITE_NOMEM; }
|
| + sParse.pConfig = pConfig;
|
| +
|
| + do {
|
| + t = fts5ExprGetToken(&sParse, &z, &token);
|
| + sqlite3Fts5Parser(pEngine, t, token, &sParse);
|
| + }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
|
| + sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
|
| +
|
| + assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
|
| + if( sParse.rc==SQLITE_OK ){
|
| + *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
|
| + if( pNew==0 ){
|
| + sParse.rc = SQLITE_NOMEM;
|
| + sqlite3Fts5ParseNodeFree(sParse.pExpr);
|
| + }else{
|
| + pNew->pRoot = sParse.pExpr;
|
| + pNew->pIndex = 0;
|
| + pNew->apExprPhrase = sParse.apPhrase;
|
| + pNew->nPhrase = sParse.nPhrase;
|
| + sParse.apPhrase = 0;
|
| + }
|
| + }
|
| +
|
| + sqlite3_free(sParse.apPhrase);
|
| + *pzErr = sParse.zErr;
|
| + return sParse.rc;
|
| +}
|
| +
|
| +/*
|
| +** Free the expression node object passed as the only argument.
|
| +*/
|
| +static void sqlite3Fts5ParseNodeFree(Fts5ExprNode *p){
|
| + if( p ){
|
| + int i;
|
| + for(i=0; i<p->nChild; i++){
|
| + sqlite3Fts5ParseNodeFree(p->apChild[i]);
|
| + }
|
| + sqlite3Fts5ParseNearsetFree(p->pNear);
|
| + sqlite3_free(p);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Free the expression object passed as the only argument.
|
| +*/
|
| +static void sqlite3Fts5ExprFree(Fts5Expr *p){
|
| + if( p ){
|
| + sqlite3Fts5ParseNodeFree(p->pRoot);
|
| + sqlite3_free(p->apExprPhrase);
|
| + sqlite3_free(p);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Argument pTerm must be a synonym iterator. Return the current rowid
|
| +** that it points to.
|
| +*/
|
| +static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
|
| + i64 iRet = 0;
|
| + int bRetValid = 0;
|
| + Fts5ExprTerm *p;
|
| +
|
| + assert( pTerm->pSynonym );
|
| + assert( bDesc==0 || bDesc==1 );
|
| + for(p=pTerm; p; p=p->pSynonym){
|
| + if( 0==sqlite3Fts5IterEof(p->pIter) ){
|
| + i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
|
| + if( bRetValid==0 || (bDesc!=(iRowid<iRet)) ){
|
| + iRet = iRowid;
|
| + bRetValid = 1;
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( pbEof && bRetValid==0 ) *pbEof = 1;
|
| + return iRet;
|
| +}
|
| +
|
| +/*
|
| +** Argument pTerm must be a synonym iterator.
|
| +*/
|
| +static int fts5ExprSynonymPoslist(
|
| + Fts5ExprTerm *pTerm,
|
| + Fts5Colset *pColset,
|
| + i64 iRowid,
|
| + int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */
|
| + u8 **pa, int *pn
|
| +){
|
| + Fts5PoslistReader aStatic[4];
|
| + Fts5PoslistReader *aIter = aStatic;
|
| + int nIter = 0;
|
| + int nAlloc = 4;
|
| + int rc = SQLITE_OK;
|
| + Fts5ExprTerm *p;
|
| +
|
| + assert( pTerm->pSynonym );
|
| + for(p=pTerm; p; p=p->pSynonym){
|
| + Fts5IndexIter *pIter = p->pIter;
|
| + if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
|
| + const u8 *a;
|
| + int n;
|
| + i64 dummy;
|
| + rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
|
| + if( rc!=SQLITE_OK ) goto synonym_poslist_out;
|
| + if( nIter==nAlloc ){
|
| + int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
|
| + Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte);
|
| + if( aNew==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + goto synonym_poslist_out;
|
| + }
|
| + memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter);
|
| + nAlloc = nAlloc*2;
|
| + if( aIter!=aStatic ) sqlite3_free(aIter);
|
| + aIter = aNew;
|
| + }
|
| + sqlite3Fts5PoslistReaderInit(a, n, &aIter[nIter]);
|
| + assert( aIter[nIter].bEof==0 );
|
| + nIter++;
|
| + }
|
| + }
|
| +
|
| + assert( *pbDel==0 );
|
| + if( nIter==1 ){
|
| + *pa = (u8*)aIter[0].a;
|
| + *pn = aIter[0].n;
|
| + }else{
|
| + Fts5PoslistWriter writer = {0};
|
| + Fts5Buffer buf = {0,0,0};
|
| + i64 iPrev = -1;
|
| + while( 1 ){
|
| + int i;
|
| + i64 iMin = FTS5_LARGEST_INT64;
|
| + for(i=0; i<nIter; i++){
|
| + if( aIter[i].bEof==0 ){
|
| + if( aIter[i].iPos==iPrev ){
|
| + if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) continue;
|
| + }
|
| + if( aIter[i].iPos<iMin ){
|
| + iMin = aIter[i].iPos;
|
| + }
|
| + }
|
| + }
|
| + if( iMin==FTS5_LARGEST_INT64 || rc!=SQLITE_OK ) break;
|
| + rc = sqlite3Fts5PoslistWriterAppend(&buf, &writer, iMin);
|
| + iPrev = iMin;
|
| + }
|
| + if( rc ){
|
| + sqlite3_free(buf.p);
|
| + }else{
|
| + *pa = buf.p;
|
| + *pn = buf.n;
|
| + *pbDel = 1;
|
| + }
|
| + }
|
| +
|
| + synonym_poslist_out:
|
| + if( aIter!=aStatic ) sqlite3_free(aIter);
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** All individual term iterators in pPhrase are guaranteed to be valid and
|
| +** pointing to the same rowid when this function is called. This function
|
| +** checks if the current rowid really is a match, and if so populates
|
| +** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch
|
| +** is set to true if this is really a match, or false otherwise.
|
| +**
|
| +** SQLITE_OK is returned if an error occurs, or an SQLite error code
|
| +** otherwise. It is not considered an error code if the current rowid is
|
| +** not a match.
|
| +*/
|
| +static int fts5ExprPhraseIsMatch(
|
| + Fts5ExprNode *pNode, /* Node pPhrase belongs to */
|
| + Fts5Colset *pColset, /* Restrict matches to these columns */
|
| + Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */
|
| + int *pbMatch /* OUT: Set to true if really a match */
|
| +){
|
| + Fts5PoslistWriter writer = {0};
|
| + Fts5PoslistReader aStatic[4];
|
| + Fts5PoslistReader *aIter = aStatic;
|
| + int i;
|
| + int rc = SQLITE_OK;
|
| +
|
| + fts5BufferZero(&pPhrase->poslist);
|
| +
|
| + /* If the aStatic[] array is not large enough, allocate a large array
|
| + ** using sqlite3_malloc(). This approach could be improved upon. */
|
| + if( pPhrase->nTerm>(int)ArraySize(aStatic) ){
|
| + int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
|
| + aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte);
|
| + if( !aIter ) return SQLITE_NOMEM;
|
| + }
|
| + memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm);
|
| +
|
| + /* Initialize a term iterator for each term in the phrase */
|
| + for(i=0; i<pPhrase->nTerm; i++){
|
| + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
|
| + i64 dummy;
|
| + int n = 0;
|
| + int bFlag = 0;
|
| + const u8 *a = 0;
|
| + if( pTerm->pSynonym ){
|
| + rc = fts5ExprSynonymPoslist(
|
| + pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
|
| + );
|
| + }else{
|
| + rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy);
|
| + }
|
| + if( rc!=SQLITE_OK ) goto ismatch_out;
|
| + sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
|
| + aIter[i].bFlag = (u8)bFlag;
|
| + if( aIter[i].bEof ) goto ismatch_out;
|
| + }
|
| +
|
| + while( 1 ){
|
| + int bMatch;
|
| + i64 iPos = aIter[0].iPos;
|
| + do {
|
| + bMatch = 1;
|
| + for(i=0; i<pPhrase->nTerm; i++){
|
| + Fts5PoslistReader *pPos = &aIter[i];
|
| + i64 iAdj = iPos + i;
|
| + if( pPos->iPos!=iAdj ){
|
| + bMatch = 0;
|
| + while( pPos->iPos<iAdj ){
|
| + if( sqlite3Fts5PoslistReaderNext(pPos) ) goto ismatch_out;
|
| + }
|
| + if( pPos->iPos>iAdj ) iPos = pPos->iPos-i;
|
| + }
|
| + }
|
| + }while( bMatch==0 );
|
| +
|
| + /* Append position iPos to the output */
|
| + rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
|
| + if( rc!=SQLITE_OK ) goto ismatch_out;
|
| +
|
| + for(i=0; i<pPhrase->nTerm; i++){
|
| + if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
|
| + }
|
| + }
|
| +
|
| + ismatch_out:
|
| + *pbMatch = (pPhrase->poslist.n>0);
|
| + for(i=0; i<pPhrase->nTerm; i++){
|
| + if( aIter[i].bFlag ) sqlite3_free((u8*)aIter[i].a);
|
| + }
|
| + if( aIter!=aStatic ) sqlite3_free(aIter);
|
| + return rc;
|
| +}
|
| +
|
| +typedef struct Fts5LookaheadReader Fts5LookaheadReader;
|
| +struct Fts5LookaheadReader {
|
| + const u8 *a; /* Buffer containing position list */
|
| + int n; /* Size of buffer a[] in bytes */
|
| + int i; /* Current offset in position list */
|
| + i64 iPos; /* Current position */
|
| + i64 iLookahead; /* Next position */
|
| +};
|
| +
|
| +#define FTS5_LOOKAHEAD_EOF (((i64)1) << 62)
|
| +
|
| +static int fts5LookaheadReaderNext(Fts5LookaheadReader *p){
|
| + p->iPos = p->iLookahead;
|
| + if( sqlite3Fts5PoslistNext64(p->a, p->n, &p->i, &p->iLookahead) ){
|
| + p->iLookahead = FTS5_LOOKAHEAD_EOF;
|
| + }
|
| + return (p->iPos==FTS5_LOOKAHEAD_EOF);
|
| +}
|
| +
|
| +static int fts5LookaheadReaderInit(
|
| + const u8 *a, int n, /* Buffer to read position list from */
|
| + Fts5LookaheadReader *p /* Iterator object to initialize */
|
| +){
|
| + memset(p, 0, sizeof(Fts5LookaheadReader));
|
| + p->a = a;
|
| + p->n = n;
|
| + fts5LookaheadReaderNext(p);
|
| + return fts5LookaheadReaderNext(p);
|
| +}
|
| +
|
| +#if 0
|
| +static int fts5LookaheadReaderEof(Fts5LookaheadReader *p){
|
| + return (p->iPos==FTS5_LOOKAHEAD_EOF);
|
| +}
|
| +#endif
|
| +
|
| +typedef struct Fts5NearTrimmer Fts5NearTrimmer;
|
| +struct Fts5NearTrimmer {
|
| + Fts5LookaheadReader reader; /* Input iterator */
|
| + Fts5PoslistWriter writer; /* Writer context */
|
| + Fts5Buffer *pOut; /* Output poslist */
|
| +};
|
| +
|
| +/*
|
| +** The near-set object passed as the first argument contains more than
|
| +** one phrase. All phrases currently point to the same row. The
|
| +** Fts5ExprPhrase.poslist buffers are populated accordingly. This function
|
| +** tests if the current row contains instances of each phrase sufficiently
|
| +** close together to meet the NEAR constraint. Non-zero is returned if it
|
| +** does, or zero otherwise.
|
| +**
|
| +** If in/out parameter (*pRc) is set to other than SQLITE_OK when this
|
| +** function is called, it is a no-op. Or, if an error (e.g. SQLITE_NOMEM)
|
| +** occurs within this function (*pRc) is set accordingly before returning.
|
| +** The return value is undefined in both these cases.
|
| +**
|
| +** If no error occurs and non-zero (a match) is returned, the position-list
|
| +** of each phrase object is edited to contain only those entries that
|
| +** meet the constraint before returning.
|
| +*/
|
| +static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){
|
| + Fts5NearTrimmer aStatic[4];
|
| + Fts5NearTrimmer *a = aStatic;
|
| + Fts5ExprPhrase **apPhrase = pNear->apPhrase;
|
| +
|
| + int i;
|
| + int rc = *pRc;
|
| + int bMatch;
|
| +
|
| + assert( pNear->nPhrase>1 );
|
| +
|
| + /* If the aStatic[] array is not large enough, allocate a large array
|
| + ** using sqlite3_malloc(). This approach could be improved upon. */
|
| + if( pNear->nPhrase>(int)ArraySize(aStatic) ){
|
| + int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
|
| + a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte);
|
| + }else{
|
| + memset(aStatic, 0, sizeof(aStatic));
|
| + }
|
| + if( rc!=SQLITE_OK ){
|
| + *pRc = rc;
|
| + return 0;
|
| + }
|
| +
|
| + /* Initialize a lookahead iterator for each phrase. After passing the
|
| + ** buffer and buffer size to the lookaside-reader init function, zero
|
| + ** the phrase poslist buffer. The new poslist for the phrase (containing
|
| + ** the same entries as the original with some entries removed on account
|
| + ** of the NEAR constraint) is written over the original even as it is
|
| + ** being read. This is safe as the entries for the new poslist are a
|
| + ** subset of the old, so it is not possible for data yet to be read to
|
| + ** be overwritten. */
|
| + for(i=0; i<pNear->nPhrase; i++){
|
| + Fts5Buffer *pPoslist = &apPhrase[i]->poslist;
|
| + fts5LookaheadReaderInit(pPoslist->p, pPoslist->n, &a[i].reader);
|
| + pPoslist->n = 0;
|
| + a[i].pOut = pPoslist;
|
| + }
|
| +
|
| + while( 1 ){
|
| + int iAdv;
|
| + i64 iMin;
|
| + i64 iMax;
|
| +
|
| + /* This block advances the phrase iterators until they point to a set of
|
| + ** entries that together comprise a match. */
|
| + iMax = a[0].reader.iPos;
|
| + do {
|
| + bMatch = 1;
|
| + for(i=0; i<pNear->nPhrase; i++){
|
| + Fts5LookaheadReader *pPos = &a[i].reader;
|
| + iMin = iMax - pNear->apPhrase[i]->nTerm - pNear->nNear;
|
| + if( pPos->iPos<iMin || pPos->iPos>iMax ){
|
| + bMatch = 0;
|
| + while( pPos->iPos<iMin ){
|
| + if( fts5LookaheadReaderNext(pPos) ) goto ismatch_out;
|
| + }
|
| + if( pPos->iPos>iMax ) iMax = pPos->iPos;
|
| + }
|
| + }
|
| + }while( bMatch==0 );
|
| +
|
| + /* Add an entry to each output position list */
|
| + for(i=0; i<pNear->nPhrase; i++){
|
| + i64 iPos = a[i].reader.iPos;
|
| + Fts5PoslistWriter *pWriter = &a[i].writer;
|
| + if( a[i].pOut->n==0 || iPos!=pWriter->iPrev ){
|
| + sqlite3Fts5PoslistWriterAppend(a[i].pOut, pWriter, iPos);
|
| + }
|
| + }
|
| +
|
| + iAdv = 0;
|
| + iMin = a[0].reader.iLookahead;
|
| + for(i=0; i<pNear->nPhrase; i++){
|
| + if( a[i].reader.iLookahead < iMin ){
|
| + iMin = a[i].reader.iLookahead;
|
| + iAdv = i;
|
| + }
|
| + }
|
| + if( fts5LookaheadReaderNext(&a[iAdv].reader) ) goto ismatch_out;
|
| + }
|
| +
|
| + ismatch_out: {
|
| + int bRet = a[0].pOut->n>0;
|
| + *pRc = rc;
|
| + if( a!=aStatic ) sqlite3_free(a);
|
| + return bRet;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Advance the first term iterator in the first phrase of pNear. Set output
|
| +** variable *pbEof to true if it reaches EOF or if an error occurs.
|
| +**
|
| +** Return SQLITE_OK if successful, or an SQLite error code if an error
|
| +** occurs.
|
| +*/
|
| +static int fts5ExprNearAdvanceFirst(
|
| + Fts5Expr *pExpr, /* Expression pPhrase belongs to */
|
| + Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */
|
| + int bFromValid,
|
| + i64 iFrom
|
| +){
|
| + Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0];
|
| + int rc = SQLITE_OK;
|
| +
|
| + if( pTerm->pSynonym ){
|
| + int bEof = 1;
|
| + Fts5ExprTerm *p;
|
| +
|
| + /* Find the firstest rowid any synonym points to. */
|
| + i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0);
|
| +
|
| + /* Advance each iterator that currently points to iRowid. Or, if iFrom
|
| + ** is valid - each iterator that points to a rowid before iFrom. */
|
| + for(p=pTerm; p; p=p->pSynonym){
|
| + if( sqlite3Fts5IterEof(p->pIter)==0 ){
|
| + i64 ii = sqlite3Fts5IterRowid(p->pIter);
|
| + if( ii==iRowid
|
| + || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc)
|
| + ){
|
| + if( bFromValid ){
|
| + rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom);
|
| + }else{
|
| + rc = sqlite3Fts5IterNext(p->pIter);
|
| + }
|
| + if( rc!=SQLITE_OK ) break;
|
| + if( sqlite3Fts5IterEof(p->pIter)==0 ){
|
| + bEof = 0;
|
| + }
|
| + }else{
|
| + bEof = 0;
|
| + }
|
| + }
|
| + }
|
| +
|
| + /* Set the EOF flag if either all synonym iterators are at EOF or an
|
| + ** error has occurred. */
|
| + pNode->bEof = (rc || bEof);
|
| + }else{
|
| + Fts5IndexIter *pIter = pTerm->pIter;
|
| +
|
| + assert( Fts5NodeIsString(pNode) );
|
| + if( bFromValid ){
|
| + rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
|
| + }else{
|
| + rc = sqlite3Fts5IterNext(pIter);
|
| + }
|
| +
|
| + pNode->bEof = (rc || sqlite3Fts5IterEof(pIter));
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Advance iterator pIter until it points to a value equal to or laster
|
| +** than the initial value of *piLast. If this means the iterator points
|
| +** to a value laster than *piLast, update *piLast to the new lastest value.
|
| +**
|
| +** If the iterator reaches EOF, set *pbEof to true before returning. If
|
| +** an error occurs, set *pRc to an error code. If either *pbEof or *pRc
|
| +** are set, return a non-zero value. Otherwise, return zero.
|
| +*/
|
| +static int fts5ExprAdvanceto(
|
| + Fts5IndexIter *pIter, /* Iterator to advance */
|
| + int bDesc, /* True if iterator is "rowid DESC" */
|
| + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */
|
| + int *pRc, /* OUT: Error code */
|
| + int *pbEof /* OUT: Set to true if EOF */
|
| +){
|
| + i64 iLast = *piLast;
|
| + i64 iRowid;
|
| +
|
| + iRowid = sqlite3Fts5IterRowid(pIter);
|
| + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
|
| + int rc = sqlite3Fts5IterNextFrom(pIter, iLast);
|
| + if( rc || sqlite3Fts5IterEof(pIter) ){
|
| + *pRc = rc;
|
| + *pbEof = 1;
|
| + return 1;
|
| + }
|
| + iRowid = sqlite3Fts5IterRowid(pIter);
|
| + assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) );
|
| + }
|
| + *piLast = iRowid;
|
| +
|
| + return 0;
|
| +}
|
| +
|
| +static int fts5ExprSynonymAdvanceto(
|
| + Fts5ExprTerm *pTerm, /* Term iterator to advance */
|
| + int bDesc, /* True if iterator is "rowid DESC" */
|
| + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */
|
| + int *pRc /* OUT: Error code */
|
| +){
|
| + int rc = SQLITE_OK;
|
| + i64 iLast = *piLast;
|
| + Fts5ExprTerm *p;
|
| + int bEof = 0;
|
| +
|
| + for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){
|
| + if( sqlite3Fts5IterEof(p->pIter)==0 ){
|
| + i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
|
| + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
|
| + rc = sqlite3Fts5IterNextFrom(p->pIter, iLast);
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + *pRc = rc;
|
| + bEof = 1;
|
| + }else{
|
| + *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof);
|
| + }
|
| + return bEof;
|
| +}
|
| +
|
| +
|
| +static int fts5ExprNearTest(
|
| + int *pRc,
|
| + Fts5Expr *pExpr, /* Expression that pNear is a part of */
|
| + Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */
|
| +){
|
| + Fts5ExprNearset *pNear = pNode->pNear;
|
| + int rc = *pRc;
|
| + int i;
|
| +
|
| + /* Check that each phrase in the nearset matches the current row.
|
| + ** Populate the pPhrase->poslist buffers at the same time. If any
|
| + ** phrase is not a match, break out of the loop early. */
|
| + for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
|
| + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
|
| + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
|
| + int bMatch = 0;
|
| + rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
|
| + if( bMatch==0 ) break;
|
| + }else{
|
| + rc = sqlite3Fts5IterPoslistBuffer(
|
| + pPhrase->aTerm[0].pIter, &pPhrase->poslist
|
| + );
|
| + }
|
| + }
|
| +
|
| + *pRc = rc;
|
| + if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
|
| + return 1;
|
| + }
|
| +
|
| + return 0;
|
| +}
|
| +
|
| +static int fts5ExprTokenTest(
|
| + Fts5Expr *pExpr, /* Expression that pNear is a part of */
|
| + Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */
|
| +){
|
| + /* As this "NEAR" object is actually a single phrase that consists
|
| + ** of a single term only, grab pointers into the poslist managed by the
|
| + ** fts5_index.c iterator object. This is much faster than synthesizing
|
| + ** a new poslist the way we have to for more complicated phrase or NEAR
|
| + ** expressions. */
|
| + Fts5ExprNearset *pNear = pNode->pNear;
|
| + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
|
| + Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
|
| + Fts5Colset *pColset = pNear->pColset;
|
| + int rc;
|
| +
|
| + assert( pNode->eType==FTS5_TERM );
|
| + assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
|
| + assert( pPhrase->aTerm[0].pSynonym==0 );
|
| +
|
| + rc = sqlite3Fts5IterPoslist(pIter, pColset,
|
| + (const u8**)&pPhrase->poslist.p, &pPhrase->poslist.n, &pNode->iRowid
|
| + );
|
| + pNode->bNomatch = (pPhrase->poslist.n==0);
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** All individual term iterators in pNear are guaranteed to be valid when
|
| +** this function is called. This function checks if all term iterators
|
| +** point to the same rowid, and if not, advances them until they do.
|
| +** If an EOF is reached before this happens, *pbEof is set to true before
|
| +** returning.
|
| +**
|
| +** SQLITE_OK is returned if an error occurs, or an SQLite error code
|
| +** otherwise. It is not considered an error code if an iterator reaches
|
| +** EOF.
|
| +*/
|
| +static int fts5ExprNearNextMatch(
|
| + Fts5Expr *pExpr, /* Expression pPhrase belongs to */
|
| + Fts5ExprNode *pNode
|
| +){
|
| + Fts5ExprNearset *pNear = pNode->pNear;
|
| + Fts5ExprPhrase *pLeft = pNear->apPhrase[0];
|
| + int rc = SQLITE_OK;
|
| + i64 iLast; /* Lastest rowid any iterator points to */
|
| + int i, j; /* Phrase and token index, respectively */
|
| + int bMatch; /* True if all terms are at the same rowid */
|
| + const int bDesc = pExpr->bDesc;
|
| +
|
| + /* Check that this node should not be FTS5_TERM */
|
| + assert( pNear->nPhrase>1
|
| + || pNear->apPhrase[0]->nTerm>1
|
| + || pNear->apPhrase[0]->aTerm[0].pSynonym
|
| + );
|
| +
|
| + /* Initialize iLast, the "lastest" rowid any iterator points to. If the
|
| + ** iterator skips through rowids in the default ascending order, this means
|
| + ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
|
| + ** means the minimum rowid. */
|
| + if( pLeft->aTerm[0].pSynonym ){
|
| + iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0);
|
| + }else{
|
| + iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter);
|
| + }
|
| +
|
| + do {
|
| + bMatch = 1;
|
| + for(i=0; i<pNear->nPhrase; i++){
|
| + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
|
| + for(j=0; j<pPhrase->nTerm; j++){
|
| + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
|
| + if( pTerm->pSynonym ){
|
| + i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
|
| + if( iRowid==iLast ) continue;
|
| + bMatch = 0;
|
| + if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
|
| + pNode->bEof = 1;
|
| + return rc;
|
| + }
|
| + }else{
|
| + Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
|
| + i64 iRowid = sqlite3Fts5IterRowid(pIter);
|
| + if( iRowid==iLast ) continue;
|
| + bMatch = 0;
|
| + if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
|
| + return rc;
|
| + }
|
| + }
|
| + }
|
| + }
|
| + }while( bMatch==0 );
|
| +
|
| + pNode->iRowid = iLast;
|
| + pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode));
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Initialize all term iterators in the pNear object. If any term is found
|
| +** to match no documents at all, return immediately without initializing any
|
| +** further iterators.
|
| +*/
|
| +static int fts5ExprNearInitAll(
|
| + Fts5Expr *pExpr,
|
| + Fts5ExprNode *pNode
|
| +){
|
| + Fts5ExprNearset *pNear = pNode->pNear;
|
| + int i, j;
|
| + int rc = SQLITE_OK;
|
| +
|
| + for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
|
| + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
|
| + for(j=0; j<pPhrase->nTerm; j++){
|
| + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
|
| + Fts5ExprTerm *p;
|
| + int bEof = 1;
|
| +
|
| + for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){
|
| + if( p->pIter ){
|
| + sqlite3Fts5IterClose(p->pIter);
|
| + p->pIter = 0;
|
| + }
|
| + rc = sqlite3Fts5IndexQuery(
|
| + pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
|
| + (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
|
| + (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
|
| + pNear->pColset,
|
| + &p->pIter
|
| + );
|
| + assert( rc==SQLITE_OK || p->pIter==0 );
|
| + if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
|
| + bEof = 0;
|
| + }
|
| + }
|
| +
|
| + if( bEof ){
|
| + pNode->bEof = 1;
|
| + return rc;
|
| + }
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/* fts5ExprNodeNext() calls fts5ExprNodeNextMatch(). And vice-versa. */
|
| +static int fts5ExprNodeNextMatch(Fts5Expr*, Fts5ExprNode*);
|
| +
|
| +
|
| +/*
|
| +** If pExpr is an ASC iterator, this function returns a value with the
|
| +** same sign as:
|
| +**
|
| +** (iLhs - iRhs)
|
| +**
|
| +** Otherwise, if this is a DESC iterator, the opposite is returned:
|
| +**
|
| +** (iRhs - iLhs)
|
| +*/
|
| +static int fts5RowidCmp(
|
| + Fts5Expr *pExpr,
|
| + i64 iLhs,
|
| + i64 iRhs
|
| +){
|
| + assert( pExpr->bDesc==0 || pExpr->bDesc==1 );
|
| + if( pExpr->bDesc==0 ){
|
| + if( iLhs<iRhs ) return -1;
|
| + return (iLhs > iRhs);
|
| + }else{
|
| + if( iLhs>iRhs ) return -1;
|
| + return (iLhs < iRhs);
|
| + }
|
| +}
|
| +
|
| +static void fts5ExprSetEof(Fts5ExprNode *pNode){
|
| + int i;
|
| + pNode->bEof = 1;
|
| + for(i=0; i<pNode->nChild; i++){
|
| + fts5ExprSetEof(pNode->apChild[i]);
|
| + }
|
| +}
|
| +
|
| +static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){
|
| + if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
|
| + Fts5ExprNearset *pNear = pNode->pNear;
|
| + int i;
|
| + for(i=0; i<pNear->nPhrase; i++){
|
| + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
|
| + pPhrase->poslist.n = 0;
|
| + }
|
| + }else{
|
| + int i;
|
| + for(i=0; i<pNode->nChild; i++){
|
| + fts5ExprNodeZeroPoslist(pNode->apChild[i]);
|
| + }
|
| + }
|
| +}
|
| +
|
| +
|
| +static int fts5ExprNodeNext(Fts5Expr*, Fts5ExprNode*, int, i64);
|
| +
|
| +/*
|
| +** Argument pNode is an FTS5_AND node.
|
| +*/
|
| +static int fts5ExprAndNextRowid(
|
| + Fts5Expr *pExpr, /* Expression pPhrase belongs to */
|
| + Fts5ExprNode *pAnd /* FTS5_AND node to advance */
|
| +){
|
| + int iChild;
|
| + i64 iLast = pAnd->iRowid;
|
| + int rc = SQLITE_OK;
|
| + int bMatch;
|
| +
|
| + assert( pAnd->bEof==0 );
|
| + do {
|
| + pAnd->bNomatch = 0;
|
| + bMatch = 1;
|
| + for(iChild=0; iChild<pAnd->nChild; iChild++){
|
| + Fts5ExprNode *pChild = pAnd->apChild[iChild];
|
| + if( 0 && pChild->eType==FTS5_STRING ){
|
| + /* TODO */
|
| + }else{
|
| + int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid);
|
| + if( cmp>0 ){
|
| + /* Advance pChild until it points to iLast or laster */
|
| + rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }
|
| + }
|
| +
|
| + /* If the child node is now at EOF, so is the parent AND node. Otherwise,
|
| + ** the child node is guaranteed to have advanced at least as far as
|
| + ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the
|
| + ** new lastest rowid seen so far. */
|
| + assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 );
|
| + if( pChild->bEof ){
|
| + fts5ExprSetEof(pAnd);
|
| + bMatch = 1;
|
| + break;
|
| + }else if( iLast!=pChild->iRowid ){
|
| + bMatch = 0;
|
| + iLast = pChild->iRowid;
|
| + }
|
| +
|
| + if( pChild->bNomatch ){
|
| + pAnd->bNomatch = 1;
|
| + }
|
| + }
|
| + }while( bMatch==0 );
|
| +
|
| + if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){
|
| + fts5ExprNodeZeroPoslist(pAnd);
|
| + }
|
| + pAnd->iRowid = iLast;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Compare the values currently indicated by the two nodes as follows:
|
| +**
|
| +** res = (*p1) - (*p2)
|
| +**
|
| +** Nodes that point to values that come later in the iteration order are
|
| +** considered to be larger. Nodes at EOF are the largest of all.
|
| +**
|
| +** This means that if the iteration order is ASC, then numerically larger
|
| +** rowids are considered larger. Or if it is the default DESC, numerically
|
| +** smaller rowids are larger.
|
| +*/
|
| +static int fts5NodeCompare(
|
| + Fts5Expr *pExpr,
|
| + Fts5ExprNode *p1,
|
| + Fts5ExprNode *p2
|
| +){
|
| + if( p2->bEof ) return -1;
|
| + if( p1->bEof ) return +1;
|
| + return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid);
|
| +}
|
| +
|
| +/*
|
| +** Advance node iterator pNode, part of expression pExpr. If argument
|
| +** bFromValid is zero, then pNode is advanced exactly once. Or, if argument
|
| +** bFromValid is non-zero, then pNode is advanced until it is at or past
|
| +** rowid value iFrom. Whether "past" means "less than" or "greater than"
|
| +** depends on whether this is an ASC or DESC iterator.
|
| +*/
|
| +static int fts5ExprNodeNext(
|
| + Fts5Expr *pExpr,
|
| + Fts5ExprNode *pNode,
|
| + int bFromValid,
|
| + i64 iFrom
|
| +){
|
| + int rc = SQLITE_OK;
|
| +
|
| + if( pNode->bEof==0 ){
|
| + switch( pNode->eType ){
|
| + case FTS5_STRING: {
|
| + rc = fts5ExprNearAdvanceFirst(pExpr, pNode, bFromValid, iFrom);
|
| + break;
|
| + };
|
| +
|
| + case FTS5_TERM: {
|
| + Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter;
|
| + if( bFromValid ){
|
| + rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
|
| + }else{
|
| + rc = sqlite3Fts5IterNext(pIter);
|
| + }
|
| + if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){
|
| + assert( rc==SQLITE_OK );
|
| + rc = fts5ExprTokenTest(pExpr, pNode);
|
| + }else{
|
| + pNode->bEof = 1;
|
| + }
|
| + return rc;
|
| + };
|
| +
|
| + case FTS5_AND: {
|
| + Fts5ExprNode *pLeft = pNode->apChild[0];
|
| + rc = fts5ExprNodeNext(pExpr, pLeft, bFromValid, iFrom);
|
| + break;
|
| + }
|
| +
|
| + case FTS5_OR: {
|
| + int i;
|
| + i64 iLast = pNode->iRowid;
|
| +
|
| + for(i=0; rc==SQLITE_OK && i<pNode->nChild; i++){
|
| + Fts5ExprNode *p1 = pNode->apChild[i];
|
| + assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 );
|
| + if( p1->bEof==0 ){
|
| + if( (p1->iRowid==iLast)
|
| + || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
|
| + ){
|
| + rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom);
|
| + }
|
| + }
|
| + }
|
| +
|
| + break;
|
| + }
|
| +
|
| + default: assert( pNode->eType==FTS5_NOT ); {
|
| + assert( pNode->nChild==2 );
|
| + rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
|
| + break;
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5ExprNodeNextMatch(pExpr, pNode);
|
| + }
|
| + }
|
| +
|
| + /* Assert that if bFromValid was true, either:
|
| + **
|
| + ** a) an error occurred, or
|
| + ** b) the node is now at EOF, or
|
| + ** c) the node is now at or past rowid iFrom.
|
| + */
|
| + assert( bFromValid==0
|
| + || rc!=SQLITE_OK /* a */
|
| + || pNode->bEof /* b */
|
| + || pNode->iRowid==iFrom || pExpr->bDesc==(pNode->iRowid<iFrom) /* c */
|
| + );
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** If pNode currently points to a match, this function returns SQLITE_OK
|
| +** without modifying it. Otherwise, pNode is advanced until it does point
|
| +** to a match or EOF is reached.
|
| +*/
|
| +static int fts5ExprNodeNextMatch(
|
| + Fts5Expr *pExpr, /* Expression of which pNode is a part */
|
| + Fts5ExprNode *pNode /* Expression node to test */
|
| +){
|
| + int rc = SQLITE_OK;
|
| + if( pNode->bEof==0 ){
|
| + switch( pNode->eType ){
|
| +
|
| + case FTS5_STRING: {
|
| + /* Advance the iterators until they all point to the same rowid */
|
| + rc = fts5ExprNearNextMatch(pExpr, pNode);
|
| + break;
|
| + }
|
| +
|
| + case FTS5_TERM: {
|
| + rc = fts5ExprTokenTest(pExpr, pNode);
|
| + break;
|
| + }
|
| +
|
| + case FTS5_AND: {
|
| + rc = fts5ExprAndNextRowid(pExpr, pNode);
|
| + break;
|
| + }
|
| +
|
| + case FTS5_OR: {
|
| + Fts5ExprNode *pNext = pNode->apChild[0];
|
| + int i;
|
| +
|
| + for(i=1; i<pNode->nChild; i++){
|
| + Fts5ExprNode *pChild = pNode->apChild[i];
|
| + int cmp = fts5NodeCompare(pExpr, pNext, pChild);
|
| + if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){
|
| + pNext = pChild;
|
| + }
|
| + }
|
| + pNode->iRowid = pNext->iRowid;
|
| + pNode->bEof = pNext->bEof;
|
| + pNode->bNomatch = pNext->bNomatch;
|
| + break;
|
| + }
|
| +
|
| + default: assert( pNode->eType==FTS5_NOT ); {
|
| + Fts5ExprNode *p1 = pNode->apChild[0];
|
| + Fts5ExprNode *p2 = pNode->apChild[1];
|
| + assert( pNode->nChild==2 );
|
| +
|
| + while( rc==SQLITE_OK && p1->bEof==0 ){
|
| + int cmp = fts5NodeCompare(pExpr, p1, p2);
|
| + if( cmp>0 ){
|
| + rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid);
|
| + cmp = fts5NodeCompare(pExpr, p1, p2);
|
| + }
|
| + assert( rc!=SQLITE_OK || cmp<=0 );
|
| + if( cmp || p2->bNomatch ) break;
|
| + rc = fts5ExprNodeNext(pExpr, p1, 0, 0);
|
| + }
|
| + pNode->bEof = p1->bEof;
|
| + pNode->iRowid = p1->iRowid;
|
| + break;
|
| + }
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Set node pNode, which is part of expression pExpr, to point to the first
|
| +** match. If there are no matches, set the Node.bEof flag to indicate EOF.
|
| +**
|
| +** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise.
|
| +** It is not an error if there are no matches.
|
| +*/
|
| +static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
|
| + int rc = SQLITE_OK;
|
| + pNode->bEof = 0;
|
| +
|
| + if( Fts5NodeIsString(pNode) ){
|
| + /* Initialize all term iterators in the NEAR object. */
|
| + rc = fts5ExprNearInitAll(pExpr, pNode);
|
| + }else{
|
| + int i;
|
| + for(i=0; i<pNode->nChild && rc==SQLITE_OK; i++){
|
| + rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]);
|
| + }
|
| + pNode->iRowid = pNode->apChild[0]->iRowid;
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5ExprNodeNextMatch(pExpr, pNode);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Begin iterating through the set of documents in index pIdx matched by
|
| +** the MATCH expression passed as the first argument. If the "bDesc"
|
| +** parameter is passed a non-zero value, iteration is in descending rowid
|
| +** order. Or, if it is zero, in ascending order.
|
| +**
|
| +** If iterating in ascending rowid order (bDesc==0), the first document
|
| +** visited is that with the smallest rowid that is larger than or equal
|
| +** to parameter iFirst. Or, if iterating in ascending order (bDesc==1),
|
| +** then the first document visited must have a rowid smaller than or
|
| +** equal to iFirst.
|
| +**
|
| +** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
|
| +** is not considered an error if the query does not match any documents.
|
| +*/
|
| +static int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){
|
| + Fts5ExprNode *pRoot = p->pRoot;
|
| + int rc = SQLITE_OK;
|
| + if( pRoot ){
|
| + p->pIndex = pIdx;
|
| + p->bDesc = bDesc;
|
| + rc = fts5ExprNodeFirst(p, pRoot);
|
| +
|
| + /* If not at EOF but the current rowid occurs earlier than iFirst in
|
| + ** the iteration order, move to document iFirst or later. */
|
| + if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
|
| + rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
|
| + }
|
| +
|
| + /* If the iterator is not at a real match, skip forward until it is. */
|
| + while( pRoot->bNomatch && rc==SQLITE_OK && pRoot->bEof==0 ){
|
| + rc = fts5ExprNodeNext(p, pRoot, 0, 0);
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Move to the next document
|
| +**
|
| +** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
|
| +** is not considered an error if the query does not match any documents.
|
| +*/
|
| +static int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){
|
| + int rc;
|
| + Fts5ExprNode *pRoot = p->pRoot;
|
| + do {
|
| + rc = fts5ExprNodeNext(p, pRoot, 0, 0);
|
| + }while( pRoot->bNomatch && pRoot->bEof==0 && rc==SQLITE_OK );
|
| + if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){
|
| + pRoot->bEof = 1;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static int sqlite3Fts5ExprEof(Fts5Expr *p){
|
| + return (p->pRoot==0 || p->pRoot->bEof);
|
| +}
|
| +
|
| +static i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
|
| + return p->pRoot->iRowid;
|
| +}
|
| +
|
| +static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
|
| + int rc = SQLITE_OK;
|
| + *pz = sqlite3Fts5Strndup(&rc, pToken->p, pToken->n);
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Free the phrase object passed as the only argument.
|
| +*/
|
| +static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
|
| + if( pPhrase ){
|
| + int i;
|
| + for(i=0; i<pPhrase->nTerm; i++){
|
| + Fts5ExprTerm *pSyn;
|
| + Fts5ExprTerm *pNext;
|
| + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
|
| + sqlite3_free(pTerm->zTerm);
|
| + sqlite3Fts5IterClose(pTerm->pIter);
|
| +
|
| + for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
|
| + pNext = pSyn->pSynonym;
|
| + sqlite3Fts5IterClose(pSyn->pIter);
|
| + sqlite3_free(pSyn);
|
| + }
|
| + }
|
| + if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist);
|
| + sqlite3_free(pPhrase);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
|
| +** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
|
| +** appended to it and the results returned.
|
| +**
|
| +** If an OOM error occurs, both the pNear and pPhrase objects are freed and
|
| +** NULL returned.
|
| +*/
|
| +static Fts5ExprNearset *sqlite3Fts5ParseNearset(
|
| + Fts5Parse *pParse, /* Parse context */
|
| + Fts5ExprNearset *pNear, /* Existing nearset, or NULL */
|
| + Fts5ExprPhrase *pPhrase /* Recently parsed phrase */
|
| +){
|
| + const int SZALLOC = 8;
|
| + Fts5ExprNearset *pRet = 0;
|
| +
|
| + if( pParse->rc==SQLITE_OK ){
|
| + if( pPhrase==0 ){
|
| + return pNear;
|
| + }
|
| + if( pNear==0 ){
|
| + int nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
|
| + pRet = sqlite3_malloc(nByte);
|
| + if( pRet==0 ){
|
| + pParse->rc = SQLITE_NOMEM;
|
| + }else{
|
| + memset(pRet, 0, nByte);
|
| + }
|
| + }else if( (pNear->nPhrase % SZALLOC)==0 ){
|
| + int nNew = pNear->nPhrase + SZALLOC;
|
| + int nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*);
|
| +
|
| + pRet = (Fts5ExprNearset*)sqlite3_realloc(pNear, nByte);
|
| + if( pRet==0 ){
|
| + pParse->rc = SQLITE_NOMEM;
|
| + }
|
| + }else{
|
| + pRet = pNear;
|
| + }
|
| + }
|
| +
|
| + if( pRet==0 ){
|
| + assert( pParse->rc!=SQLITE_OK );
|
| + sqlite3Fts5ParseNearsetFree(pNear);
|
| + sqlite3Fts5ParsePhraseFree(pPhrase);
|
| + }else{
|
| + pRet->apPhrase[pRet->nPhrase++] = pPhrase;
|
| + }
|
| + return pRet;
|
| +}
|
| +
|
| +typedef struct TokenCtx TokenCtx;
|
| +struct TokenCtx {
|
| + Fts5ExprPhrase *pPhrase;
|
| + int rc;
|
| +};
|
| +
|
| +/*
|
| +** Callback for tokenizing terms used by ParseTerm().
|
| +*/
|
| +static int fts5ParseTokenize(
|
| + void *pContext, /* Pointer to Fts5InsertCtx object */
|
| + int tflags, /* Mask of FTS5_TOKEN_* flags */
|
| + const char *pToken, /* Buffer containing token */
|
| + int nToken, /* Size of token in bytes */
|
| + int iUnused1, /* Start offset of token */
|
| + int iUnused2 /* End offset of token */
|
| +){
|
| + int rc = SQLITE_OK;
|
| + const int SZALLOC = 8;
|
| + TokenCtx *pCtx = (TokenCtx*)pContext;
|
| + Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
|
| +
|
| + /* If an error has already occurred, this is a no-op */
|
| + if( pCtx->rc!=SQLITE_OK ) return pCtx->rc;
|
| +
|
| + assert( pPhrase==0 || pPhrase->nTerm>0 );
|
| + if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){
|
| + Fts5ExprTerm *pSyn;
|
| + int nByte = sizeof(Fts5ExprTerm) + nToken+1;
|
| + pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
|
| + if( pSyn==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + memset(pSyn, 0, nByte);
|
| + pSyn->zTerm = (char*)&pSyn[1];
|
| + memcpy(pSyn->zTerm, pToken, nToken);
|
| + pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
|
| + pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
|
| + }
|
| + }else{
|
| + Fts5ExprTerm *pTerm;
|
| + if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
|
| + Fts5ExprPhrase *pNew;
|
| + int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0);
|
| +
|
| + pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase,
|
| + sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew
|
| + );
|
| + if( pNew==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase));
|
| + pCtx->pPhrase = pPhrase = pNew;
|
| + pNew->nTerm = nNew - SZALLOC;
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
|
| + memset(pTerm, 0, sizeof(Fts5ExprTerm));
|
| + pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
|
| + }
|
| + }
|
| +
|
| + pCtx->rc = rc;
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Free the phrase object passed as the only argument.
|
| +*/
|
| +static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase *pPhrase){
|
| + fts5ExprPhraseFree(pPhrase);
|
| +}
|
| +
|
| +/*
|
| +** Free the phrase object passed as the second argument.
|
| +*/
|
| +static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset *pNear){
|
| + if( pNear ){
|
| + int i;
|
| + for(i=0; i<pNear->nPhrase; i++){
|
| + fts5ExprPhraseFree(pNear->apPhrase[i]);
|
| + }
|
| + sqlite3_free(pNear->pColset);
|
| + sqlite3_free(pNear);
|
| + }
|
| +}
|
| +
|
| +static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p){
|
| + assert( pParse->pExpr==0 );
|
| + pParse->pExpr = p;
|
| +}
|
| +
|
| +/*
|
| +** This function is called by the parser to process a string token. The
|
| +** string may or may not be quoted. In any case it is tokenized and a
|
| +** phrase object consisting of all tokens returned.
|
| +*/
|
| +static Fts5ExprPhrase *sqlite3Fts5ParseTerm(
|
| + Fts5Parse *pParse, /* Parse context */
|
| + Fts5ExprPhrase *pAppend, /* Phrase to append to */
|
| + Fts5Token *pToken, /* String to tokenize */
|
| + int bPrefix /* True if there is a trailing "*" */
|
| +){
|
| + Fts5Config *pConfig = pParse->pConfig;
|
| + TokenCtx sCtx; /* Context object passed to callback */
|
| + int rc; /* Tokenize return code */
|
| + char *z = 0;
|
| +
|
| + memset(&sCtx, 0, sizeof(TokenCtx));
|
| + sCtx.pPhrase = pAppend;
|
| +
|
| + rc = fts5ParseStringFromToken(pToken, &z);
|
| + if( rc==SQLITE_OK ){
|
| + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0);
|
| + int n;
|
| + sqlite3Fts5Dequote(z);
|
| + n = (int)strlen(z);
|
| + rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
|
| + }
|
| + sqlite3_free(z);
|
| + if( rc || (rc = sCtx.rc) ){
|
| + pParse->rc = rc;
|
| + fts5ExprPhraseFree(sCtx.pPhrase);
|
| + sCtx.pPhrase = 0;
|
| + }else if( sCtx.pPhrase ){
|
| +
|
| + if( pAppend==0 ){
|
| + if( (pParse->nPhrase % 8)==0 ){
|
| + int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
|
| + Fts5ExprPhrase **apNew;
|
| + apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
|
| + if( apNew==0 ){
|
| + pParse->rc = SQLITE_NOMEM;
|
| + fts5ExprPhraseFree(sCtx.pPhrase);
|
| + return 0;
|
| + }
|
| + pParse->apPhrase = apNew;
|
| + }
|
| + pParse->nPhrase++;
|
| + }
|
| +
|
| + pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
|
| + assert( sCtx.pPhrase->nTerm>0 );
|
| + sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
|
| + }
|
| +
|
| + return sCtx.pPhrase;
|
| +}
|
| +
|
| +/*
|
| +** Create a new FTS5 expression by cloning phrase iPhrase of the
|
| +** expression passed as the second argument.
|
| +*/
|
| +static int sqlite3Fts5ExprClonePhrase(
|
| + Fts5Config *pConfig,
|
| + Fts5Expr *pExpr,
|
| + int iPhrase,
|
| + Fts5Expr **ppNew
|
| +){
|
| + int rc = SQLITE_OK; /* Return code */
|
| + Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */
|
| + int i; /* Used to iterate through phrase terms */
|
| +
|
| + Fts5Expr *pNew = 0; /* Expression to return via *ppNew */
|
| +
|
| + TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */
|
| +
|
| +
|
| + pOrig = pExpr->apExprPhrase[iPhrase];
|
| +
|
| + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
|
| + if( rc==SQLITE_OK ){
|
| + pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc,
|
| + sizeof(Fts5ExprPhrase*));
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc,
|
| + sizeof(Fts5ExprNode));
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc,
|
| + sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*));
|
| + }
|
| +
|
| + for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
|
| + int tflags = 0;
|
| + Fts5ExprTerm *p;
|
| + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
|
| + const char *zTerm = p->zTerm;
|
| + rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
|
| + 0, 0);
|
| + tflags = FTS5_TOKEN_COLOCATED;
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + /* All the allocations succeeded. Put the expression object together. */
|
| + pNew->pIndex = pExpr->pIndex;
|
| + pNew->nPhrase = 1;
|
| + pNew->apExprPhrase[0] = sCtx.pPhrase;
|
| + pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
|
| + pNew->pRoot->pNear->nPhrase = 1;
|
| + sCtx.pPhrase->pNode = pNew->pRoot;
|
| +
|
| + if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
|
| + pNew->pRoot->eType = FTS5_TERM;
|
| + }else{
|
| + pNew->pRoot->eType = FTS5_STRING;
|
| + }
|
| + }else{
|
| + sqlite3Fts5ExprFree(pNew);
|
| + fts5ExprPhraseFree(sCtx.pPhrase);
|
| + pNew = 0;
|
| + }
|
| +
|
| + *ppNew = pNew;
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Token pTok has appeared in a MATCH expression where the NEAR operator
|
| +** is expected. If token pTok does not contain "NEAR", store an error
|
| +** in the pParse object.
|
| +*/
|
| +static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token *pTok){
|
| + if( pTok->n!=4 || memcmp("NEAR", pTok->p, 4) ){
|
| + sqlite3Fts5ParseError(
|
| + pParse, "fts5: syntax error near \"%.*s\"", pTok->n, pTok->p
|
| + );
|
| + }
|
| +}
|
| +
|
| +static void sqlite3Fts5ParseSetDistance(
|
| + Fts5Parse *pParse,
|
| + Fts5ExprNearset *pNear,
|
| + Fts5Token *p
|
| +){
|
| + int nNear = 0;
|
| + int i;
|
| + if( p->n ){
|
| + for(i=0; i<p->n; i++){
|
| + char c = (char)p->p[i];
|
| + if( c<'0' || c>'9' ){
|
| + sqlite3Fts5ParseError(
|
| + pParse, "expected integer, got \"%.*s\"", p->n, p->p
|
| + );
|
| + return;
|
| + }
|
| + nNear = nNear * 10 + (p->p[i] - '0');
|
| + }
|
| + }else{
|
| + nNear = FTS5_DEFAULT_NEARDIST;
|
| + }
|
| + pNear->nNear = nNear;
|
| +}
|
| +
|
| +/*
|
| +** The second argument passed to this function may be NULL, or it may be
|
| +** an existing Fts5Colset object. This function returns a pointer to
|
| +** a new colset object containing the contents of (p) with new value column
|
| +** number iCol appended.
|
| +**
|
| +** If an OOM error occurs, store an error code in pParse and return NULL.
|
| +** The old colset object (if any) is not freed in this case.
|
| +*/
|
| +static Fts5Colset *fts5ParseColset(
|
| + Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */
|
| + Fts5Colset *p, /* Existing colset object */
|
| + int iCol /* New column to add to colset object */
|
| +){
|
| + int nCol = p ? p->nCol : 0; /* Num. columns already in colset object */
|
| + Fts5Colset *pNew; /* New colset object to return */
|
| +
|
| + assert( pParse->rc==SQLITE_OK );
|
| + assert( iCol>=0 && iCol<pParse->pConfig->nCol );
|
| +
|
| + pNew = sqlite3_realloc(p, sizeof(Fts5Colset) + sizeof(int)*nCol);
|
| + if( pNew==0 ){
|
| + pParse->rc = SQLITE_NOMEM;
|
| + }else{
|
| + int *aiCol = pNew->aiCol;
|
| + int i, j;
|
| + for(i=0; i<nCol; i++){
|
| + if( aiCol[i]==iCol ) return pNew;
|
| + if( aiCol[i]>iCol ) break;
|
| + }
|
| + for(j=nCol; j>i; j--){
|
| + aiCol[j] = aiCol[j-1];
|
| + }
|
| + aiCol[i] = iCol;
|
| + pNew->nCol = nCol+1;
|
| +
|
| +#ifndef NDEBUG
|
| + /* Check that the array is in order and contains no duplicate entries. */
|
| + for(i=1; i<pNew->nCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] );
|
| +#endif
|
| + }
|
| +
|
| + return pNew;
|
| +}
|
| +
|
| +static Fts5Colset *sqlite3Fts5ParseColset(
|
| + Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */
|
| + Fts5Colset *pColset, /* Existing colset object */
|
| + Fts5Token *p
|
| +){
|
| + Fts5Colset *pRet = 0;
|
| + int iCol;
|
| + char *z; /* Dequoted copy of token p */
|
| +
|
| + z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n);
|
| + if( pParse->rc==SQLITE_OK ){
|
| + Fts5Config *pConfig = pParse->pConfig;
|
| + sqlite3Fts5Dequote(z);
|
| + for(iCol=0; iCol<pConfig->nCol; iCol++){
|
| + if( 0==sqlite3_stricmp(pConfig->azCol[iCol], z) ) break;
|
| + }
|
| + if( iCol==pConfig->nCol ){
|
| + sqlite3Fts5ParseError(pParse, "no such column: %s", z);
|
| + }else{
|
| + pRet = fts5ParseColset(pParse, pColset, iCol);
|
| + }
|
| + sqlite3_free(z);
|
| + }
|
| +
|
| + if( pRet==0 ){
|
| + assert( pParse->rc!=SQLITE_OK );
|
| + sqlite3_free(pColset);
|
| + }
|
| +
|
| + return pRet;
|
| +}
|
| +
|
| +static void sqlite3Fts5ParseSetColset(
|
| + Fts5Parse *pParse,
|
| + Fts5ExprNearset *pNear,
|
| + Fts5Colset *pColset
|
| +){
|
| + if( pNear ){
|
| + pNear->pColset = pColset;
|
| + }else{
|
| + sqlite3_free(pColset);
|
| + }
|
| +}
|
| +
|
| +static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
|
| + if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){
|
| + int nByte = sizeof(Fts5ExprNode*) * pSub->nChild;
|
| + memcpy(&p->apChild[p->nChild], pSub->apChild, nByte);
|
| + p->nChild += pSub->nChild;
|
| + sqlite3_free(pSub);
|
| + }else{
|
| + p->apChild[p->nChild++] = pSub;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Allocate and return a new expression object. If anything goes wrong (i.e.
|
| +** OOM error), leave an error code in pParse and return NULL.
|
| +*/
|
| +static Fts5ExprNode *sqlite3Fts5ParseNode(
|
| + Fts5Parse *pParse, /* Parse context */
|
| + int eType, /* FTS5_STRING, AND, OR or NOT */
|
| + Fts5ExprNode *pLeft, /* Left hand child expression */
|
| + Fts5ExprNode *pRight, /* Right hand child expression */
|
| + Fts5ExprNearset *pNear /* For STRING expressions, the near cluster */
|
| +){
|
| + Fts5ExprNode *pRet = 0;
|
| +
|
| + if( pParse->rc==SQLITE_OK ){
|
| + int nChild = 0; /* Number of children of returned node */
|
| + int nByte; /* Bytes of space to allocate for this node */
|
| +
|
| + assert( (eType!=FTS5_STRING && !pNear)
|
| + || (eType==FTS5_STRING && !pLeft && !pRight)
|
| + );
|
| + if( eType==FTS5_STRING && pNear==0 ) return 0;
|
| + if( eType!=FTS5_STRING && pLeft==0 ) return pRight;
|
| + if( eType!=FTS5_STRING && pRight==0 ) return pLeft;
|
| +
|
| + if( eType==FTS5_NOT ){
|
| + nChild = 2;
|
| + }else if( eType==FTS5_AND || eType==FTS5_OR ){
|
| + nChild = 2;
|
| + if( pLeft->eType==eType ) nChild += pLeft->nChild-1;
|
| + if( pRight->eType==eType ) nChild += pRight->nChild-1;
|
| + }
|
| +
|
| + nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1);
|
| + pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte);
|
| +
|
| + if( pRet ){
|
| + pRet->eType = eType;
|
| + pRet->pNear = pNear;
|
| + if( eType==FTS5_STRING ){
|
| + int iPhrase;
|
| + for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
|
| + pNear->apPhrase[iPhrase]->pNode = pRet;
|
| + }
|
| + if( pNear->nPhrase==1
|
| + && pNear->apPhrase[0]->nTerm==1
|
| + && pNear->apPhrase[0]->aTerm[0].pSynonym==0
|
| + ){
|
| + pRet->eType = FTS5_TERM;
|
| + }
|
| + }else{
|
| + fts5ExprAddChildren(pRet, pLeft);
|
| + fts5ExprAddChildren(pRet, pRight);
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( pRet==0 ){
|
| + assert( pParse->rc!=SQLITE_OK );
|
| + sqlite3Fts5ParseNodeFree(pLeft);
|
| + sqlite3Fts5ParseNodeFree(pRight);
|
| + sqlite3Fts5ParseNearsetFree(pNear);
|
| + }
|
| + return pRet;
|
| +}
|
| +
|
| +static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
|
| + int nByte = 0;
|
| + Fts5ExprTerm *p;
|
| + char *zQuoted;
|
| +
|
| + /* Determine the maximum amount of space required. */
|
| + for(p=pTerm; p; p=p->pSynonym){
|
| + nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2;
|
| + }
|
| + zQuoted = sqlite3_malloc(nByte);
|
| +
|
| + if( zQuoted ){
|
| + int i = 0;
|
| + for(p=pTerm; p; p=p->pSynonym){
|
| + char *zIn = p->zTerm;
|
| + zQuoted[i++] = '"';
|
| + while( *zIn ){
|
| + if( *zIn=='"' ) zQuoted[i++] = '"';
|
| + zQuoted[i++] = *zIn++;
|
| + }
|
| + zQuoted[i++] = '"';
|
| + if( p->pSynonym ) zQuoted[i++] = '|';
|
| + }
|
| + if( pTerm->bPrefix ){
|
| + zQuoted[i++] = ' ';
|
| + zQuoted[i++] = '*';
|
| + }
|
| + zQuoted[i++] = '\0';
|
| + }
|
| + return zQuoted;
|
| +}
|
| +
|
| +static char *fts5PrintfAppend(char *zApp, const char *zFmt, ...){
|
| + char *zNew;
|
| + va_list ap;
|
| + va_start(ap, zFmt);
|
| + zNew = sqlite3_vmprintf(zFmt, ap);
|
| + va_end(ap);
|
| + if( zApp && zNew ){
|
| + char *zNew2 = sqlite3_mprintf("%s%s", zApp, zNew);
|
| + sqlite3_free(zNew);
|
| + zNew = zNew2;
|
| + }
|
| + sqlite3_free(zApp);
|
| + return zNew;
|
| +}
|
| +
|
| +/*
|
| +** Compose a tcl-readable representation of expression pExpr. Return a
|
| +** pointer to a buffer containing that representation. It is the
|
| +** responsibility of the caller to at some point free the buffer using
|
| +** sqlite3_free().
|
| +*/
|
| +static char *fts5ExprPrintTcl(
|
| + Fts5Config *pConfig,
|
| + const char *zNearsetCmd,
|
| + Fts5ExprNode *pExpr
|
| +){
|
| + char *zRet = 0;
|
| + if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){
|
| + Fts5ExprNearset *pNear = pExpr->pNear;
|
| + int i;
|
| + int iTerm;
|
| +
|
| + zRet = fts5PrintfAppend(zRet, "%s ", zNearsetCmd);
|
| + if( zRet==0 ) return 0;
|
| + if( pNear->pColset ){
|
| + int *aiCol = pNear->pColset->aiCol;
|
| + int nCol = pNear->pColset->nCol;
|
| + if( nCol==1 ){
|
| + zRet = fts5PrintfAppend(zRet, "-col %d ", aiCol[0]);
|
| + }else{
|
| + zRet = fts5PrintfAppend(zRet, "-col {%d", aiCol[0]);
|
| + for(i=1; i<pNear->pColset->nCol; i++){
|
| + zRet = fts5PrintfAppend(zRet, " %d", aiCol[i]);
|
| + }
|
| + zRet = fts5PrintfAppend(zRet, "} ");
|
| + }
|
| + if( zRet==0 ) return 0;
|
| + }
|
| +
|
| + if( pNear->nPhrase>1 ){
|
| + zRet = fts5PrintfAppend(zRet, "-near %d ", pNear->nNear);
|
| + if( zRet==0 ) return 0;
|
| + }
|
| +
|
| + zRet = fts5PrintfAppend(zRet, "--");
|
| + if( zRet==0 ) return 0;
|
| +
|
| + for(i=0; i<pNear->nPhrase; i++){
|
| + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
|
| +
|
| + zRet = fts5PrintfAppend(zRet, " {");
|
| + for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
|
| + char *zTerm = pPhrase->aTerm[iTerm].zTerm;
|
| + zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
|
| + }
|
| +
|
| + if( zRet ) zRet = fts5PrintfAppend(zRet, "}");
|
| + if( zRet==0 ) return 0;
|
| + }
|
| +
|
| + }else{
|
| + char const *zOp = 0;
|
| + int i;
|
| + switch( pExpr->eType ){
|
| + case FTS5_AND: zOp = "AND"; break;
|
| + case FTS5_NOT: zOp = "NOT"; break;
|
| + default:
|
| + assert( pExpr->eType==FTS5_OR );
|
| + zOp = "OR";
|
| + break;
|
| + }
|
| +
|
| + zRet = sqlite3_mprintf("%s", zOp);
|
| + for(i=0; zRet && i<pExpr->nChild; i++){
|
| + char *z = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->apChild[i]);
|
| + if( !z ){
|
| + sqlite3_free(zRet);
|
| + zRet = 0;
|
| + }else{
|
| + zRet = fts5PrintfAppend(zRet, " [%z]", z);
|
| + }
|
| + }
|
| + }
|
| +
|
| + return zRet;
|
| +}
|
| +
|
| +static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){
|
| + char *zRet = 0;
|
| + if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){
|
| + Fts5ExprNearset *pNear = pExpr->pNear;
|
| + int i;
|
| + int iTerm;
|
| +
|
| + if( pNear->pColset ){
|
| + int iCol = pNear->pColset->aiCol[0];
|
| + zRet = fts5PrintfAppend(zRet, "%s : ", pConfig->azCol[iCol]);
|
| + if( zRet==0 ) return 0;
|
| + }
|
| +
|
| + if( pNear->nPhrase>1 ){
|
| + zRet = fts5PrintfAppend(zRet, "NEAR(");
|
| + if( zRet==0 ) return 0;
|
| + }
|
| +
|
| + for(i=0; i<pNear->nPhrase; i++){
|
| + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
|
| + if( i!=0 ){
|
| + zRet = fts5PrintfAppend(zRet, " ");
|
| + if( zRet==0 ) return 0;
|
| + }
|
| + for(iTerm=0; iTerm<pPhrase->nTerm; iTerm++){
|
| + char *zTerm = fts5ExprTermPrint(&pPhrase->aTerm[iTerm]);
|
| + if( zTerm ){
|
| + zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" + ", zTerm);
|
| + sqlite3_free(zTerm);
|
| + }
|
| + if( zTerm==0 || zRet==0 ){
|
| + sqlite3_free(zRet);
|
| + return 0;
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( pNear->nPhrase>1 ){
|
| + zRet = fts5PrintfAppend(zRet, ", %d)", pNear->nNear);
|
| + if( zRet==0 ) return 0;
|
| + }
|
| +
|
| + }else{
|
| + char const *zOp = 0;
|
| + int i;
|
| +
|
| + switch( pExpr->eType ){
|
| + case FTS5_AND: zOp = " AND "; break;
|
| + case FTS5_NOT: zOp = " NOT "; break;
|
| + default:
|
| + assert( pExpr->eType==FTS5_OR );
|
| + zOp = " OR ";
|
| + break;
|
| + }
|
| +
|
| + for(i=0; i<pExpr->nChild; i++){
|
| + char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]);
|
| + if( z==0 ){
|
| + sqlite3_free(zRet);
|
| + zRet = 0;
|
| + }else{
|
| + int e = pExpr->apChild[i]->eType;
|
| + int b = (e!=FTS5_STRING && e!=FTS5_TERM);
|
| + zRet = fts5PrintfAppend(zRet, "%s%s%z%s",
|
| + (i==0 ? "" : zOp),
|
| + (b?"(":""), z, (b?")":"")
|
| + );
|
| + }
|
| + if( zRet==0 ) break;
|
| + }
|
| + }
|
| +
|
| + return zRet;
|
| +}
|
| +
|
| +/*
|
| +** The implementation of user-defined scalar functions fts5_expr() (bTcl==0)
|
| +** and fts5_expr_tcl() (bTcl!=0).
|
| +*/
|
| +static void fts5ExprFunction(
|
| + sqlite3_context *pCtx, /* Function call context */
|
| + int nArg, /* Number of args */
|
| + sqlite3_value **apVal, /* Function arguments */
|
| + int bTcl
|
| +){
|
| + Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
|
| + sqlite3 *db = sqlite3_context_db_handle(pCtx);
|
| + const char *zExpr = 0;
|
| + char *zErr = 0;
|
| + Fts5Expr *pExpr = 0;
|
| + int rc;
|
| + int i;
|
| +
|
| + const char **azConfig; /* Array of arguments for Fts5Config */
|
| + const char *zNearsetCmd = "nearset";
|
| + int nConfig; /* Size of azConfig[] */
|
| + Fts5Config *pConfig = 0;
|
| + int iArg = 1;
|
| +
|
| + if( nArg<1 ){
|
| + zErr = sqlite3_mprintf("wrong number of arguments to function %s",
|
| + bTcl ? "fts5_expr_tcl" : "fts5_expr"
|
| + );
|
| + sqlite3_result_error(pCtx, zErr, -1);
|
| + sqlite3_free(zErr);
|
| + return;
|
| + }
|
| +
|
| + if( bTcl && nArg>1 ){
|
| + zNearsetCmd = (const char*)sqlite3_value_text(apVal[1]);
|
| + iArg = 2;
|
| + }
|
| +
|
| + nConfig = 3 + (nArg-iArg);
|
| + azConfig = (const char**)sqlite3_malloc(sizeof(char*) * nConfig);
|
| + if( azConfig==0 ){
|
| + sqlite3_result_error_nomem(pCtx);
|
| + return;
|
| + }
|
| + azConfig[0] = 0;
|
| + azConfig[1] = "main";
|
| + azConfig[2] = "tbl";
|
| + for(i=3; iArg<nArg; iArg++){
|
| + azConfig[i++] = (const char*)sqlite3_value_text(apVal[iArg]);
|
| + }
|
| +
|
| + zExpr = (const char*)sqlite3_value_text(apVal[0]);
|
| +
|
| + rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + char *zText;
|
| + if( pExpr->pRoot==0 ){
|
| + zText = sqlite3_mprintf("");
|
| + }else if( bTcl ){
|
| + zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
|
| + }else{
|
| + zText = fts5ExprPrint(pConfig, pExpr->pRoot);
|
| + }
|
| + if( zText==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT);
|
| + sqlite3_free(zText);
|
| + }
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + if( zErr ){
|
| + sqlite3_result_error(pCtx, zErr, -1);
|
| + sqlite3_free(zErr);
|
| + }else{
|
| + sqlite3_result_error_code(pCtx, rc);
|
| + }
|
| + }
|
| + sqlite3_free((void *)azConfig);
|
| + sqlite3Fts5ConfigFree(pConfig);
|
| + sqlite3Fts5ExprFree(pExpr);
|
| +}
|
| +
|
| +static void fts5ExprFunctionHr(
|
| + sqlite3_context *pCtx, /* Function call context */
|
| + int nArg, /* Number of args */
|
| + sqlite3_value **apVal /* Function arguments */
|
| +){
|
| + fts5ExprFunction(pCtx, nArg, apVal, 0);
|
| +}
|
| +static void fts5ExprFunctionTcl(
|
| + sqlite3_context *pCtx, /* Function call context */
|
| + int nArg, /* Number of args */
|
| + sqlite3_value **apVal /* Function arguments */
|
| +){
|
| + fts5ExprFunction(pCtx, nArg, apVal, 1);
|
| +}
|
| +
|
| +/*
|
| +** The implementation of an SQLite user-defined-function that accepts a
|
| +** single integer as an argument. If the integer is an alpha-numeric
|
| +** unicode code point, 1 is returned. Otherwise 0.
|
| +*/
|
| +static void fts5ExprIsAlnum(
|
| + sqlite3_context *pCtx, /* Function call context */
|
| + int nArg, /* Number of args */
|
| + sqlite3_value **apVal /* Function arguments */
|
| +){
|
| + int iCode;
|
| + if( nArg!=1 ){
|
| + sqlite3_result_error(pCtx,
|
| + "wrong number of arguments to function fts5_isalnum", -1
|
| + );
|
| + return;
|
| + }
|
| + iCode = sqlite3_value_int(apVal[0]);
|
| + sqlite3_result_int(pCtx, sqlite3Fts5UnicodeIsalnum(iCode));
|
| +}
|
| +
|
| +static void fts5ExprFold(
|
| + sqlite3_context *pCtx, /* Function call context */
|
| + int nArg, /* Number of args */
|
| + sqlite3_value **apVal /* Function arguments */
|
| +){
|
| + if( nArg!=1 && nArg!=2 ){
|
| + sqlite3_result_error(pCtx,
|
| + "wrong number of arguments to function fts5_fold", -1
|
| + );
|
| + }else{
|
| + int iCode;
|
| + int bRemoveDiacritics = 0;
|
| + iCode = sqlite3_value_int(apVal[0]);
|
| + if( nArg==2 ) bRemoveDiacritics = sqlite3_value_int(apVal[1]);
|
| + sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics));
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** This is called during initialization to register the fts5_expr() scalar
|
| +** UDF with the SQLite handle passed as the only argument.
|
| +*/
|
| +static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
|
| + struct Fts5ExprFunc {
|
| + const char *z;
|
| + void (*x)(sqlite3_context*,int,sqlite3_value**);
|
| + } aFunc[] = {
|
| + { "fts5_expr", fts5ExprFunctionHr },
|
| + { "fts5_expr_tcl", fts5ExprFunctionTcl },
|
| + { "fts5_isalnum", fts5ExprIsAlnum },
|
| + { "fts5_fold", fts5ExprFold },
|
| + };
|
| + int i;
|
| + int rc = SQLITE_OK;
|
| + void *pCtx = (void*)pGlobal;
|
| +
|
| + for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aFunc); i++){
|
| + struct Fts5ExprFunc *p = &aFunc[i];
|
| + rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
|
| + }
|
| +
|
| + /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */
|
| +#ifndef NDEBUG
|
| + (void)sqlite3Fts5ParserTrace;
|
| +#endif
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Return the number of phrases in expression pExpr.
|
| +*/
|
| +static int sqlite3Fts5ExprPhraseCount(Fts5Expr *pExpr){
|
| + return (pExpr ? pExpr->nPhrase : 0);
|
| +}
|
| +
|
| +/*
|
| +** Return the number of terms in the iPhrase'th phrase in pExpr.
|
| +*/
|
| +static int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){
|
| + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0;
|
| + return pExpr->apExprPhrase[iPhrase]->nTerm;
|
| +}
|
| +
|
| +/*
|
| +** This function is used to access the current position list for phrase
|
| +** iPhrase.
|
| +*/
|
| +static int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){
|
| + int nRet;
|
| + Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
|
| + Fts5ExprNode *pNode = pPhrase->pNode;
|
| + if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){
|
| + *pa = pPhrase->poslist.p;
|
| + nRet = pPhrase->poslist.n;
|
| + }else{
|
| + *pa = 0;
|
| + nRet = 0;
|
| + }
|
| + return nRet;
|
| +}
|
| +
|
| +/*
|
| +** 2014 August 11
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +*/
|
| +
|
| +
|
| +
|
| +/* #include "fts5Int.h" */
|
| +
|
| +typedef struct Fts5HashEntry Fts5HashEntry;
|
| +
|
| +/*
|
| +** This file contains the implementation of an in-memory hash table used
|
| +** to accumuluate "term -> doclist" content before it is flused to a level-0
|
| +** segment.
|
| +*/
|
| +
|
| +
|
| +struct Fts5Hash {
|
| + int *pnByte; /* Pointer to bytes counter */
|
| + int nEntry; /* Number of entries currently in hash */
|
| + int nSlot; /* Size of aSlot[] array */
|
| + Fts5HashEntry *pScan; /* Current ordered scan item */
|
| + Fts5HashEntry **aSlot; /* Array of hash slots */
|
| +};
|
| +
|
| +/*
|
| +** Each entry in the hash table is represented by an object of the
|
| +** following type. Each object, its key (zKey[]) and its current data
|
| +** are stored in a single memory allocation. The position list data
|
| +** immediately follows the key data in memory.
|
| +**
|
| +** The data that follows the key is in a similar, but not identical format
|
| +** to the doclist data stored in the database. It is:
|
| +**
|
| +** * Rowid, as a varint
|
| +** * Position list, without 0x00 terminator.
|
| +** * Size of previous position list and rowid, as a 4 byte
|
| +** big-endian integer.
|
| +**
|
| +** iRowidOff:
|
| +** Offset of last rowid written to data area. Relative to first byte of
|
| +** structure.
|
| +**
|
| +** nData:
|
| +** Bytes of data written since iRowidOff.
|
| +*/
|
| +struct Fts5HashEntry {
|
| + Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */
|
| + Fts5HashEntry *pScanNext; /* Next entry in sorted order */
|
| +
|
| + int nAlloc; /* Total size of allocation */
|
| + int iSzPoslist; /* Offset of space for 4-byte poslist size */
|
| + int nData; /* Total bytes of data (incl. structure) */
|
| + u8 bDel; /* Set delete-flag @ iSzPoslist */
|
| +
|
| + int iCol; /* Column of last value written */
|
| + int iPos; /* Position of last value written */
|
| + i64 iRowid; /* Rowid of last value written */
|
| + char zKey[8]; /* Nul-terminated entry key */
|
| +};
|
| +
|
| +/*
|
| +** Size of Fts5HashEntry without the zKey[] array.
|
| +*/
|
| +#define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8)
|
| +
|
| +
|
| +
|
| +/*
|
| +** Allocate a new hash table.
|
| +*/
|
| +static int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){
|
| + int rc = SQLITE_OK;
|
| + Fts5Hash *pNew;
|
| +
|
| + *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash));
|
| + if( pNew==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + int nByte;
|
| + memset(pNew, 0, sizeof(Fts5Hash));
|
| + pNew->pnByte = pnByte;
|
| +
|
| + pNew->nSlot = 1024;
|
| + nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
|
| + pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte);
|
| + if( pNew->aSlot==0 ){
|
| + sqlite3_free(pNew);
|
| + *ppNew = 0;
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + memset(pNew->aSlot, 0, nByte);
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Free a hash table object.
|
| +*/
|
| +static void sqlite3Fts5HashFree(Fts5Hash *pHash){
|
| + if( pHash ){
|
| + sqlite3Fts5HashClear(pHash);
|
| + sqlite3_free(pHash->aSlot);
|
| + sqlite3_free(pHash);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Empty (but do not delete) a hash table.
|
| +*/
|
| +static void sqlite3Fts5HashClear(Fts5Hash *pHash){
|
| + int i;
|
| + for(i=0; i<pHash->nSlot; i++){
|
| + Fts5HashEntry *pNext;
|
| + Fts5HashEntry *pSlot;
|
| + for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){
|
| + pNext = pSlot->pHashNext;
|
| + sqlite3_free(pSlot);
|
| + }
|
| + }
|
| + memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*));
|
| + pHash->nEntry = 0;
|
| +}
|
| +
|
| +static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){
|
| + int i;
|
| + unsigned int h = 13;
|
| + for(i=n-1; i>=0; i--){
|
| + h = (h << 3) ^ h ^ p[i];
|
| + }
|
| + return (h % nSlot);
|
| +}
|
| +
|
| +static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){
|
| + int i;
|
| + unsigned int h = 13;
|
| + for(i=n-1; i>=0; i--){
|
| + h = (h << 3) ^ h ^ p[i];
|
| + }
|
| + h = (h << 3) ^ h ^ b;
|
| + return (h % nSlot);
|
| +}
|
| +
|
| +/*
|
| +** Resize the hash table by doubling the number of slots.
|
| +*/
|
| +static int fts5HashResize(Fts5Hash *pHash){
|
| + int nNew = pHash->nSlot*2;
|
| + int i;
|
| + Fts5HashEntry **apNew;
|
| + Fts5HashEntry **apOld = pHash->aSlot;
|
| +
|
| + apNew = (Fts5HashEntry**)sqlite3_malloc(nNew*sizeof(Fts5HashEntry*));
|
| + if( !apNew ) return SQLITE_NOMEM;
|
| + memset(apNew, 0, nNew*sizeof(Fts5HashEntry*));
|
| +
|
| + for(i=0; i<pHash->nSlot; i++){
|
| + while( apOld[i] ){
|
| + int iHash;
|
| + Fts5HashEntry *p = apOld[i];
|
| + apOld[i] = p->pHashNext;
|
| + iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey));
|
| + p->pHashNext = apNew[iHash];
|
| + apNew[iHash] = p;
|
| + }
|
| + }
|
| +
|
| + sqlite3_free(apOld);
|
| + pHash->nSlot = nNew;
|
| + pHash->aSlot = apNew;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static void fts5HashAddPoslistSize(Fts5HashEntry *p){
|
| + if( p->iSzPoslist ){
|
| + u8 *pPtr = (u8*)p;
|
| + int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */
|
| + int nPos = nSz*2 + p->bDel; /* Value of nPos field */
|
| +
|
| + assert( p->bDel==0 || p->bDel==1 );
|
| + if( nPos<=127 ){
|
| + pPtr[p->iSzPoslist] = (u8)nPos;
|
| + }else{
|
| + int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
|
| + memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
|
| + sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
|
| + p->nData += (nByte-1);
|
| + }
|
| + p->bDel = 0;
|
| + p->iSzPoslist = 0;
|
| + }
|
| +}
|
| +
|
| +static int sqlite3Fts5HashWrite(
|
| + Fts5Hash *pHash,
|
| + i64 iRowid, /* Rowid for this entry */
|
| + int iCol, /* Column token appears in (-ve -> delete) */
|
| + int iPos, /* Position of token within column */
|
| + char bByte, /* First byte of token */
|
| + const char *pToken, int nToken /* Token to add or remove to or from index */
|
| +){
|
| + unsigned int iHash;
|
| + Fts5HashEntry *p;
|
| + u8 *pPtr;
|
| + int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */
|
| +
|
| + /* Attempt to locate an existing hash entry */
|
| + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
|
| + for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
|
| + if( p->zKey[0]==bByte
|
| + && memcmp(&p->zKey[1], pToken, nToken)==0
|
| + && p->zKey[nToken+1]==0
|
| + ){
|
| + break;
|
| + }
|
| + }
|
| +
|
| + /* If an existing hash entry cannot be found, create a new one. */
|
| + if( p==0 ){
|
| + int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
|
| + if( nByte<128 ) nByte = 128;
|
| +
|
| + if( (pHash->nEntry*2)>=pHash->nSlot ){
|
| + int rc = fts5HashResize(pHash);
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
|
| + }
|
| +
|
| + p = (Fts5HashEntry*)sqlite3_malloc(nByte);
|
| + if( !p ) return SQLITE_NOMEM;
|
| + memset(p, 0, FTS5_HASHENTRYSIZE);
|
| + p->nAlloc = nByte;
|
| + p->zKey[0] = bByte;
|
| + memcpy(&p->zKey[1], pToken, nToken);
|
| + assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
|
| + p->zKey[nToken+1] = '\0';
|
| + p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
|
| + p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
|
| + p->iSzPoslist = p->nData;
|
| + p->nData += 1;
|
| + p->iRowid = iRowid;
|
| + p->pHashNext = pHash->aSlot[iHash];
|
| + pHash->aSlot[iHash] = p;
|
| + pHash->nEntry++;
|
| + nIncr += p->nData;
|
| + }
|
| +
|
| + /* Check there is enough space to append a new entry. Worst case scenario
|
| + ** is:
|
| + **
|
| + ** + 9 bytes for a new rowid,
|
| + ** + 4 byte reserved for the "poslist size" varint.
|
| + ** + 1 byte for a "new column" byte,
|
| + ** + 3 bytes for a new column number (16-bit max) as a varint,
|
| + ** + 5 bytes for the new position offset (32-bit max).
|
| + */
|
| + if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
|
| + int nNew = p->nAlloc * 2;
|
| + Fts5HashEntry *pNew;
|
| + Fts5HashEntry **pp;
|
| + pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
|
| + if( pNew==0 ) return SQLITE_NOMEM;
|
| + pNew->nAlloc = nNew;
|
| + for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
|
| + *pp = pNew;
|
| + p = pNew;
|
| + }
|
| + pPtr = (u8*)p;
|
| + nIncr -= p->nData;
|
| +
|
| + /* If this is a new rowid, append the 4-byte size field for the previous
|
| + ** entry, and the new rowid for this entry. */
|
| + if( iRowid!=p->iRowid ){
|
| + fts5HashAddPoslistSize(p);
|
| + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
|
| + p->iSzPoslist = p->nData;
|
| + p->nData += 1;
|
| + p->iCol = 0;
|
| + p->iPos = 0;
|
| + p->iRowid = iRowid;
|
| + }
|
| +
|
| + if( iCol>=0 ){
|
| + /* Append a new column value, if necessary */
|
| + assert( iCol>=p->iCol );
|
| + if( iCol!=p->iCol ){
|
| + pPtr[p->nData++] = 0x01;
|
| + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
|
| + p->iCol = iCol;
|
| + p->iPos = 0;
|
| + }
|
| +
|
| + /* Append the new position offset */
|
| + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
|
| + p->iPos = iPos;
|
| + }else{
|
| + /* This is a delete. Set the delete flag. */
|
| + p->bDel = 1;
|
| + }
|
| + nIncr += p->nData;
|
| +
|
| + *pHash->pnByte += nIncr;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Arguments pLeft and pRight point to linked-lists of hash-entry objects,
|
| +** each sorted in key order. This function merges the two lists into a
|
| +** single list and returns a pointer to its first element.
|
| +*/
|
| +static Fts5HashEntry *fts5HashEntryMerge(
|
| + Fts5HashEntry *pLeft,
|
| + Fts5HashEntry *pRight
|
| +){
|
| + Fts5HashEntry *p1 = pLeft;
|
| + Fts5HashEntry *p2 = pRight;
|
| + Fts5HashEntry *pRet = 0;
|
| + Fts5HashEntry **ppOut = &pRet;
|
| +
|
| + while( p1 || p2 ){
|
| + if( p1==0 ){
|
| + *ppOut = p2;
|
| + p2 = 0;
|
| + }else if( p2==0 ){
|
| + *ppOut = p1;
|
| + p1 = 0;
|
| + }else{
|
| + int i = 0;
|
| + while( p1->zKey[i]==p2->zKey[i] ) i++;
|
| +
|
| + if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){
|
| + /* p2 is smaller */
|
| + *ppOut = p2;
|
| + ppOut = &p2->pScanNext;
|
| + p2 = p2->pScanNext;
|
| + }else{
|
| + /* p1 is smaller */
|
| + *ppOut = p1;
|
| + ppOut = &p1->pScanNext;
|
| + p1 = p1->pScanNext;
|
| + }
|
| + *ppOut = 0;
|
| + }
|
| + }
|
| +
|
| + return pRet;
|
| +}
|
| +
|
| +/*
|
| +** Extract all tokens from hash table iHash and link them into a list
|
| +** in sorted order. The hash table is cleared before returning. It is
|
| +** the responsibility of the caller to free the elements of the returned
|
| +** list.
|
| +*/
|
| +static int fts5HashEntrySort(
|
| + Fts5Hash *pHash,
|
| + const char *pTerm, int nTerm, /* Query prefix, if any */
|
| + Fts5HashEntry **ppSorted
|
| +){
|
| + const int nMergeSlot = 32;
|
| + Fts5HashEntry **ap;
|
| + Fts5HashEntry *pList;
|
| + int iSlot;
|
| + int i;
|
| +
|
| + *ppSorted = 0;
|
| + ap = sqlite3_malloc(sizeof(Fts5HashEntry*) * nMergeSlot);
|
| + if( !ap ) return SQLITE_NOMEM;
|
| + memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot);
|
| +
|
| + for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
|
| + Fts5HashEntry *pIter;
|
| + for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
|
| + if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){
|
| + Fts5HashEntry *pEntry = pIter;
|
| + pEntry->pScanNext = 0;
|
| + for(i=0; ap[i]; i++){
|
| + pEntry = fts5HashEntryMerge(pEntry, ap[i]);
|
| + ap[i] = 0;
|
| + }
|
| + ap[i] = pEntry;
|
| + }
|
| + }
|
| + }
|
| +
|
| + pList = 0;
|
| + for(i=0; i<nMergeSlot; i++){
|
| + pList = fts5HashEntryMerge(pList, ap[i]);
|
| + }
|
| +
|
| + pHash->nEntry = 0;
|
| + sqlite3_free(ap);
|
| + *ppSorted = pList;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Query the hash table for a doclist associated with term pTerm/nTerm.
|
| +*/
|
| +static int sqlite3Fts5HashQuery(
|
| + Fts5Hash *pHash, /* Hash table to query */
|
| + const char *pTerm, int nTerm, /* Query term */
|
| + const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */
|
| + int *pnDoclist /* OUT: Size of doclist in bytes */
|
| +){
|
| + unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
|
| + Fts5HashEntry *p;
|
| +
|
| + for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
|
| + if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break;
|
| + }
|
| +
|
| + if( p ){
|
| + fts5HashAddPoslistSize(p);
|
| + *ppDoclist = (const u8*)&p->zKey[nTerm+1];
|
| + *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
|
| + }else{
|
| + *ppDoclist = 0;
|
| + *pnDoclist = 0;
|
| + }
|
| +
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static int sqlite3Fts5HashScanInit(
|
| + Fts5Hash *p, /* Hash table to query */
|
| + const char *pTerm, int nTerm /* Query prefix */
|
| +){
|
| + return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
|
| +}
|
| +
|
| +static void sqlite3Fts5HashScanNext(Fts5Hash *p){
|
| + assert( !sqlite3Fts5HashScanEof(p) );
|
| + p->pScan = p->pScan->pScanNext;
|
| +}
|
| +
|
| +static int sqlite3Fts5HashScanEof(Fts5Hash *p){
|
| + return (p->pScan==0);
|
| +}
|
| +
|
| +static void sqlite3Fts5HashScanEntry(
|
| + Fts5Hash *pHash,
|
| + const char **pzTerm, /* OUT: term (nul-terminated) */
|
| + const u8 **ppDoclist, /* OUT: pointer to doclist */
|
| + int *pnDoclist /* OUT: size of doclist in bytes */
|
| +){
|
| + Fts5HashEntry *p;
|
| + if( (p = pHash->pScan) ){
|
| + int nTerm = (int)strlen(p->zKey);
|
| + fts5HashAddPoslistSize(p);
|
| + *pzTerm = p->zKey;
|
| + *ppDoclist = (const u8*)&p->zKey[nTerm+1];
|
| + *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
|
| + }else{
|
| + *pzTerm = 0;
|
| + *ppDoclist = 0;
|
| + *pnDoclist = 0;
|
| + }
|
| +}
|
| +
|
| +
|
| +/*
|
| +** 2014 May 31
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +** Low level access to the FTS index stored in the database file. The
|
| +** routines in this file file implement all read and write access to the
|
| +** %_data table. Other parts of the system access this functionality via
|
| +** the interface defined in fts5Int.h.
|
| +*/
|
| +
|
| +
|
| +/* #include "fts5Int.h" */
|
| +
|
| +/*
|
| +** Overview:
|
| +**
|
| +** The %_data table contains all the FTS indexes for an FTS5 virtual table.
|
| +** As well as the main term index, there may be up to 31 prefix indexes.
|
| +** The format is similar to FTS3/4, except that:
|
| +**
|
| +** * all segment b-tree leaf data is stored in fixed size page records
|
| +** (e.g. 1000 bytes). A single doclist may span multiple pages. Care is
|
| +** taken to ensure it is possible to iterate in either direction through
|
| +** the entries in a doclist, or to seek to a specific entry within a
|
| +** doclist, without loading it into memory.
|
| +**
|
| +** * large doclists that span many pages have associated "doclist index"
|
| +** records that contain a copy of the first rowid on each page spanned by
|
| +** the doclist. This is used to speed up seek operations, and merges of
|
| +** large doclists with very small doclists.
|
| +**
|
| +** * extra fields in the "structure record" record the state of ongoing
|
| +** incremental merge operations.
|
| +**
|
| +*/
|
| +
|
| +
|
| +#define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */
|
| +#define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */
|
| +
|
| +#define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */
|
| +
|
| +#define FTS5_MAIN_PREFIX '0'
|
| +
|
| +#if FTS5_MAX_PREFIX_INDEXES > 31
|
| +# error "FTS5_MAX_PREFIX_INDEXES is too large"
|
| +#endif
|
| +
|
| +/*
|
| +** Details:
|
| +**
|
| +** The %_data table managed by this module,
|
| +**
|
| +** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB);
|
| +**
|
| +** , contains the following 5 types of records. See the comments surrounding
|
| +** the FTS5_*_ROWID macros below for a description of how %_data rowids are
|
| +** assigned to each fo them.
|
| +**
|
| +** 1. Structure Records:
|
| +**
|
| +** The set of segments that make up an index - the index structure - are
|
| +** recorded in a single record within the %_data table. The record consists
|
| +** of a single 32-bit configuration cookie value followed by a list of
|
| +** SQLite varints. If the FTS table features more than one index (because
|
| +** there are one or more prefix indexes), it is guaranteed that all share
|
| +** the same cookie value.
|
| +**
|
| +** Immediately following the configuration cookie, the record begins with
|
| +** three varints:
|
| +**
|
| +** + number of levels,
|
| +** + total number of segments on all levels,
|
| +** + value of write counter.
|
| +**
|
| +** Then, for each level from 0 to nMax:
|
| +**
|
| +** + number of input segments in ongoing merge.
|
| +** + total number of segments in level.
|
| +** + for each segment from oldest to newest:
|
| +** + segment id (always > 0)
|
| +** + first leaf page number (often 1, always greater than 0)
|
| +** + final leaf page number
|
| +**
|
| +** 2. The Averages Record:
|
| +**
|
| +** A single record within the %_data table. The data is a list of varints.
|
| +** The first value is the number of rows in the index. Then, for each column
|
| +** from left to right, the total number of tokens in the column for all
|
| +** rows of the table.
|
| +**
|
| +** 3. Segment leaves:
|
| +**
|
| +** TERM/DOCLIST FORMAT:
|
| +**
|
| +** Most of each segment leaf is taken up by term/doclist data. The
|
| +** general format of term/doclist, starting with the first term
|
| +** on the leaf page, is:
|
| +**
|
| +** varint : size of first term
|
| +** blob: first term data
|
| +** doclist: first doclist
|
| +** zero-or-more {
|
| +** varint: number of bytes in common with previous term
|
| +** varint: number of bytes of new term data (nNew)
|
| +** blob: nNew bytes of new term data
|
| +** doclist: next doclist
|
| +** }
|
| +**
|
| +** doclist format:
|
| +**
|
| +** varint: first rowid
|
| +** poslist: first poslist
|
| +** zero-or-more {
|
| +** varint: rowid delta (always > 0)
|
| +** poslist: next poslist
|
| +** }
|
| +**
|
| +** poslist format:
|
| +**
|
| +** varint: size of poslist in bytes multiplied by 2, not including
|
| +** this field. Plus 1 if this entry carries the "delete" flag.
|
| +** collist: collist for column 0
|
| +** zero-or-more {
|
| +** 0x01 byte
|
| +** varint: column number (I)
|
| +** collist: collist for column I
|
| +** }
|
| +**
|
| +** collist format:
|
| +**
|
| +** varint: first offset + 2
|
| +** zero-or-more {
|
| +** varint: offset delta + 2
|
| +** }
|
| +**
|
| +** PAGE FORMAT
|
| +**
|
| +** Each leaf page begins with a 4-byte header containing 2 16-bit
|
| +** unsigned integer fields in big-endian format. They are:
|
| +**
|
| +** * The byte offset of the first rowid on the page, if it exists
|
| +** and occurs before the first term (otherwise 0).
|
| +**
|
| +** * The byte offset of the start of the page footer. If the page
|
| +** footer is 0 bytes in size, then this field is the same as the
|
| +** size of the leaf page in bytes.
|
| +**
|
| +** The page footer consists of a single varint for each term located
|
| +** on the page. Each varint is the byte offset of the current term
|
| +** within the page, delta-compressed against the previous value. In
|
| +** other words, the first varint in the footer is the byte offset of
|
| +** the first term, the second is the byte offset of the second less that
|
| +** of the first, and so on.
|
| +**
|
| +** The term/doclist format described above is accurate if the entire
|
| +** term/doclist data fits on a single leaf page. If this is not the case,
|
| +** the format is changed in two ways:
|
| +**
|
| +** + if the first rowid on a page occurs before the first term, it
|
| +** is stored as a literal value:
|
| +**
|
| +** varint: first rowid
|
| +**
|
| +** + the first term on each page is stored in the same way as the
|
| +** very first term of the segment:
|
| +**
|
| +** varint : size of first term
|
| +** blob: first term data
|
| +**
|
| +** 5. Segment doclist indexes:
|
| +**
|
| +** Doclist indexes are themselves b-trees, however they usually consist of
|
| +** a single leaf record only. The format of each doclist index leaf page
|
| +** is:
|
| +**
|
| +** * Flags byte. Bits are:
|
| +** 0x01: Clear if leaf is also the root page, otherwise set.
|
| +**
|
| +** * Page number of fts index leaf page. As a varint.
|
| +**
|
| +** * First rowid on page indicated by previous field. As a varint.
|
| +**
|
| +** * A list of varints, one for each subsequent termless page. A
|
| +** positive delta if the termless page contains at least one rowid,
|
| +** or an 0x00 byte otherwise.
|
| +**
|
| +** Internal doclist index nodes are:
|
| +**
|
| +** * Flags byte. Bits are:
|
| +** 0x01: Clear for root page, otherwise set.
|
| +**
|
| +** * Page number of first child page. As a varint.
|
| +**
|
| +** * Copy of first rowid on page indicated by previous field. As a varint.
|
| +**
|
| +** * A list of delta-encoded varints - the first rowid on each subsequent
|
| +** child page.
|
| +**
|
| +*/
|
| +
|
| +/*
|
| +** Rowids for the averages and structure records in the %_data table.
|
| +*/
|
| +#define FTS5_AVERAGES_ROWID 1 /* Rowid used for the averages record */
|
| +#define FTS5_STRUCTURE_ROWID 10 /* The structure record */
|
| +
|
| +/*
|
| +** Macros determining the rowids used by segment leaves and dlidx leaves
|
| +** and nodes. All nodes and leaves are stored in the %_data table with large
|
| +** positive rowids.
|
| +**
|
| +** Each segment has a unique non-zero 16-bit id.
|
| +**
|
| +** The rowid for each segment leaf is found by passing the segment id and
|
| +** the leaf page number to the FTS5_SEGMENT_ROWID macro. Leaves are numbered
|
| +** sequentially starting from 1.
|
| +*/
|
| +#define FTS5_DATA_ID_B 16 /* Max seg id number 65535 */
|
| +#define FTS5_DATA_DLI_B 1 /* Doclist-index flag (1 bit) */
|
| +#define FTS5_DATA_HEIGHT_B 5 /* Max dlidx tree height of 32 */
|
| +#define FTS5_DATA_PAGE_B 31 /* Max page number of 2147483648 */
|
| +
|
| +#define fts5_dri(segid, dlidx, height, pgno) ( \
|
| + ((i64)(segid) << (FTS5_DATA_PAGE_B+FTS5_DATA_HEIGHT_B+FTS5_DATA_DLI_B)) + \
|
| + ((i64)(dlidx) << (FTS5_DATA_PAGE_B + FTS5_DATA_HEIGHT_B)) + \
|
| + ((i64)(height) << (FTS5_DATA_PAGE_B)) + \
|
| + ((i64)(pgno)) \
|
| +)
|
| +
|
| +#define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno)
|
| +#define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno)
|
| +
|
| +/*
|
| +** Maximum segments permitted in a single index
|
| +*/
|
| +#define FTS5_MAX_SEGMENT 2000
|
| +
|
| +#ifdef SQLITE_DEBUG
|
| +static int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
|
| +#endif
|
| +
|
| +
|
| +/*
|
| +** Each time a blob is read from the %_data table, it is padded with this
|
| +** many zero bytes. This makes it easier to decode the various record formats
|
| +** without overreading if the records are corrupt.
|
| +*/
|
| +#define FTS5_DATA_ZERO_PADDING 8
|
| +#define FTS5_DATA_PADDING 20
|
| +
|
| +typedef struct Fts5Data Fts5Data;
|
| +typedef struct Fts5DlidxIter Fts5DlidxIter;
|
| +typedef struct Fts5DlidxLvl Fts5DlidxLvl;
|
| +typedef struct Fts5DlidxWriter Fts5DlidxWriter;
|
| +typedef struct Fts5PageWriter Fts5PageWriter;
|
| +typedef struct Fts5SegIter Fts5SegIter;
|
| +typedef struct Fts5DoclistIter Fts5DoclistIter;
|
| +typedef struct Fts5SegWriter Fts5SegWriter;
|
| +typedef struct Fts5Structure Fts5Structure;
|
| +typedef struct Fts5StructureLevel Fts5StructureLevel;
|
| +typedef struct Fts5StructureSegment Fts5StructureSegment;
|
| +
|
| +struct Fts5Data {
|
| + u8 *p; /* Pointer to buffer containing record */
|
| + int nn; /* Size of record in bytes */
|
| + int szLeaf; /* Size of leaf without page-index */
|
| +};
|
| +
|
| +/*
|
| +** One object per %_data table.
|
| +*/
|
| +struct Fts5Index {
|
| + Fts5Config *pConfig; /* Virtual table configuration */
|
| + char *zDataTbl; /* Name of %_data table */
|
| + int nWorkUnit; /* Leaf pages in a "unit" of work */
|
| +
|
| + /*
|
| + ** Variables related to the accumulation of tokens and doclists within the
|
| + ** in-memory hash tables before they are flushed to disk.
|
| + */
|
| + Fts5Hash *pHash; /* Hash table for in-memory data */
|
| + int nPendingData; /* Current bytes of pending data */
|
| + i64 iWriteRowid; /* Rowid for current doc being written */
|
| + int bDelete; /* Current write is a delete */
|
| +
|
| + /* Error state. */
|
| + int rc; /* Current error code */
|
| +
|
| + /* State used by the fts5DataXXX() functions. */
|
| + sqlite3_blob *pReader; /* RO incr-blob open on %_data table */
|
| + sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */
|
| + sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */
|
| + sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */
|
| + sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */
|
| + sqlite3_stmt *pIdxSelect;
|
| + int nRead; /* Total number of blocks read */
|
| +};
|
| +
|
| +struct Fts5DoclistIter {
|
| + u8 *aEof; /* Pointer to 1 byte past end of doclist */
|
| +
|
| + /* Output variables. aPoslist==0 at EOF */
|
| + i64 iRowid;
|
| + u8 *aPoslist;
|
| + int nPoslist;
|
| + int nSize;
|
| +};
|
| +
|
| +/*
|
| +** The contents of the "structure" record for each index are represented
|
| +** using an Fts5Structure record in memory. Which uses instances of the
|
| +** other Fts5StructureXXX types as components.
|
| +*/
|
| +struct Fts5StructureSegment {
|
| + int iSegid; /* Segment id */
|
| + int pgnoFirst; /* First leaf page number in segment */
|
| + int pgnoLast; /* Last leaf page number in segment */
|
| +};
|
| +struct Fts5StructureLevel {
|
| + int nMerge; /* Number of segments in incr-merge */
|
| + int nSeg; /* Total number of segments on level */
|
| + Fts5StructureSegment *aSeg; /* Array of segments. aSeg[0] is oldest. */
|
| +};
|
| +struct Fts5Structure {
|
| + int nRef; /* Object reference count */
|
| + u64 nWriteCounter; /* Total leaves written to level 0 */
|
| + int nSegment; /* Total segments in this structure */
|
| + int nLevel; /* Number of levels in this index */
|
| + Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */
|
| +};
|
| +
|
| +/*
|
| +** An object of type Fts5SegWriter is used to write to segments.
|
| +*/
|
| +struct Fts5PageWriter {
|
| + int pgno; /* Page number for this page */
|
| + int iPrevPgidx; /* Previous value written into pgidx */
|
| + Fts5Buffer buf; /* Buffer containing leaf data */
|
| + Fts5Buffer pgidx; /* Buffer containing page-index */
|
| + Fts5Buffer term; /* Buffer containing previous term on page */
|
| +};
|
| +struct Fts5DlidxWriter {
|
| + int pgno; /* Page number for this page */
|
| + int bPrevValid; /* True if iPrev is valid */
|
| + i64 iPrev; /* Previous rowid value written to page */
|
| + Fts5Buffer buf; /* Buffer containing page data */
|
| +};
|
| +struct Fts5SegWriter {
|
| + int iSegid; /* Segid to write to */
|
| + Fts5PageWriter writer; /* PageWriter object */
|
| + i64 iPrevRowid; /* Previous rowid written to current leaf */
|
| + u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */
|
| + u8 bFirstRowidInPage; /* True if next rowid is first in page */
|
| + /* TODO1: Can use (writer.pgidx.n==0) instead of bFirstTermInPage */
|
| + u8 bFirstTermInPage; /* True if next term will be first in leaf */
|
| + int nLeafWritten; /* Number of leaf pages written */
|
| + int nEmpty; /* Number of contiguous term-less nodes */
|
| +
|
| + int nDlidx; /* Allocated size of aDlidx[] array */
|
| + Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */
|
| +
|
| + /* Values to insert into the %_idx table */
|
| + Fts5Buffer btterm; /* Next term to insert into %_idx table */
|
| + int iBtPage; /* Page number corresponding to btterm */
|
| +};
|
| +
|
| +typedef struct Fts5CResult Fts5CResult;
|
| +struct Fts5CResult {
|
| + u16 iFirst; /* aSeg[] index of firstest iterator */
|
| + u8 bTermEq; /* True if the terms are equal */
|
| +};
|
| +
|
| +/*
|
| +** Object for iterating through a single segment, visiting each term/rowid
|
| +** pair in the segment.
|
| +**
|
| +** pSeg:
|
| +** The segment to iterate through.
|
| +**
|
| +** iLeafPgno:
|
| +** Current leaf page number within segment.
|
| +**
|
| +** iLeafOffset:
|
| +** Byte offset within the current leaf that is the first byte of the
|
| +** position list data (one byte passed the position-list size field).
|
| +** rowid field of the current entry. Usually this is the size field of the
|
| +** position list data. The exception is if the rowid for the current entry
|
| +** is the last thing on the leaf page.
|
| +**
|
| +** pLeaf:
|
| +** Buffer containing current leaf page data. Set to NULL at EOF.
|
| +**
|
| +** iTermLeafPgno, iTermLeafOffset:
|
| +** Leaf page number containing the last term read from the segment. And
|
| +** the offset immediately following the term data.
|
| +**
|
| +** flags:
|
| +** Mask of FTS5_SEGITER_XXX values. Interpreted as follows:
|
| +**
|
| +** FTS5_SEGITER_ONETERM:
|
| +** If set, set the iterator to point to EOF after the current doclist
|
| +** has been exhausted. Do not proceed to the next term in the segment.
|
| +**
|
| +** FTS5_SEGITER_REVERSE:
|
| +** This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If
|
| +** it is set, iterate through rowid in descending order instead of the
|
| +** default ascending order.
|
| +**
|
| +** iRowidOffset/nRowidOffset/aRowidOffset:
|
| +** These are used if the FTS5_SEGITER_REVERSE flag is set.
|
| +**
|
| +** For each rowid on the page corresponding to the current term, the
|
| +** corresponding aRowidOffset[] entry is set to the byte offset of the
|
| +** start of the "position-list-size" field within the page.
|
| +**
|
| +** iTermIdx:
|
| +** Index of current term on iTermLeafPgno.
|
| +*/
|
| +struct Fts5SegIter {
|
| + Fts5StructureSegment *pSeg; /* Segment to iterate through */
|
| + int flags; /* Mask of configuration flags */
|
| + int iLeafPgno; /* Current leaf page number */
|
| + Fts5Data *pLeaf; /* Current leaf data */
|
| + Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */
|
| + int iLeafOffset; /* Byte offset within current leaf */
|
| +
|
| + /* The page and offset from which the current term was read. The offset
|
| + ** is the offset of the first rowid in the current doclist. */
|
| + int iTermLeafPgno;
|
| + int iTermLeafOffset;
|
| +
|
| + int iPgidxOff; /* Next offset in pgidx */
|
| + int iEndofDoclist;
|
| +
|
| + /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */
|
| + int iRowidOffset; /* Current entry in aRowidOffset[] */
|
| + int nRowidOffset; /* Allocated size of aRowidOffset[] array */
|
| + int *aRowidOffset; /* Array of offset to rowid fields */
|
| +
|
| + Fts5DlidxIter *pDlidx; /* If there is a doclist-index */
|
| +
|
| + /* Variables populated based on current entry. */
|
| + Fts5Buffer term; /* Current term */
|
| + i64 iRowid; /* Current rowid */
|
| + int nPos; /* Number of bytes in current position list */
|
| + int bDel; /* True if the delete flag is set */
|
| +};
|
| +
|
| +/*
|
| +** Argument is a pointer to an Fts5Data structure that contains a
|
| +** leaf page.
|
| +*/
|
| +#define ASSERT_SZLEAF_OK(x) assert( \
|
| + (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \
|
| +)
|
| +
|
| +#define FTS5_SEGITER_ONETERM 0x01
|
| +#define FTS5_SEGITER_REVERSE 0x02
|
| +
|
| +
|
| +/*
|
| +** Argument is a pointer to an Fts5Data structure that contains a leaf
|
| +** page. This macro evaluates to true if the leaf contains no terms, or
|
| +** false if it contains at least one term.
|
| +*/
|
| +#define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn)
|
| +
|
| +#define fts5LeafTermOff(x, i) (fts5GetU16(&(x)->p[(x)->szLeaf + (i)*2]))
|
| +
|
| +#define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p))
|
| +
|
| +/*
|
| +** Object for iterating through the merged results of one or more segments,
|
| +** visiting each term/rowid pair in the merged data.
|
| +**
|
| +** nSeg is always a power of two greater than or equal to the number of
|
| +** segments that this object is merging data from. Both the aSeg[] and
|
| +** aFirst[] arrays are sized at nSeg entries. The aSeg[] array is padded
|
| +** with zeroed objects - these are handled as if they were iterators opened
|
| +** on empty segments.
|
| +**
|
| +** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an
|
| +** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the
|
| +** comparison in this context is the index of the iterator that currently
|
| +** points to the smaller term/rowid combination. Iterators at EOF are
|
| +** considered to be greater than all other iterators.
|
| +**
|
| +** aFirst[1] contains the index in aSeg[] of the iterator that points to
|
| +** the smallest key overall. aFirst[0] is unused.
|
| +**
|
| +** poslist:
|
| +** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
|
| +** There is no way to tell if this is populated or not.
|
| +*/
|
| +struct Fts5IndexIter {
|
| + Fts5Index *pIndex; /* Index that owns this iterator */
|
| + Fts5Structure *pStruct; /* Database structure for this iterator */
|
| + Fts5Buffer poslist; /* Buffer containing current poslist */
|
| +
|
| + int nSeg; /* Size of aSeg[] array */
|
| + int bRev; /* True to iterate in reverse order */
|
| + u8 bSkipEmpty; /* True to skip deleted entries */
|
| + u8 bEof; /* True at EOF */
|
| + u8 bFiltered; /* True if column-filter already applied */
|
| +
|
| + i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */
|
| + Fts5CResult *aFirst; /* Current merge state (see above) */
|
| + Fts5SegIter aSeg[1]; /* Array of segment iterators */
|
| +};
|
| +
|
| +
|
| +/*
|
| +** An instance of the following type is used to iterate through the contents
|
| +** of a doclist-index record.
|
| +**
|
| +** pData:
|
| +** Record containing the doclist-index data.
|
| +**
|
| +** bEof:
|
| +** Set to true once iterator has reached EOF.
|
| +**
|
| +** iOff:
|
| +** Set to the current offset within record pData.
|
| +*/
|
| +struct Fts5DlidxLvl {
|
| + Fts5Data *pData; /* Data for current page of this level */
|
| + int iOff; /* Current offset into pData */
|
| + int bEof; /* At EOF already */
|
| + int iFirstOff; /* Used by reverse iterators */
|
| +
|
| + /* Output variables */
|
| + int iLeafPgno; /* Page number of current leaf page */
|
| + i64 iRowid; /* First rowid on leaf iLeafPgno */
|
| +};
|
| +struct Fts5DlidxIter {
|
| + int nLvl;
|
| + int iSegid;
|
| + Fts5DlidxLvl aLvl[1];
|
| +};
|
| +
|
| +static void fts5PutU16(u8 *aOut, u16 iVal){
|
| + aOut[0] = (iVal>>8);
|
| + aOut[1] = (iVal&0xFF);
|
| +}
|
| +
|
| +static u16 fts5GetU16(const u8 *aIn){
|
| + return ((u16)aIn[0] << 8) + aIn[1];
|
| +}
|
| +
|
| +/*
|
| +** Allocate and return a buffer at least nByte bytes in size.
|
| +**
|
| +** If an OOM error is encountered, return NULL and set the error code in
|
| +** the Fts5Index handle passed as the first argument.
|
| +*/
|
| +static void *fts5IdxMalloc(Fts5Index *p, int nByte){
|
| + return sqlite3Fts5MallocZero(&p->rc, nByte);
|
| +}
|
| +
|
| +/*
|
| +** Compare the contents of the pLeft buffer with the pRight/nRight blob.
|
| +**
|
| +** Return -ve if pLeft is smaller than pRight, 0 if they are equal or
|
| +** +ve if pRight is smaller than pLeft. In other words:
|
| +**
|
| +** res = *pLeft - *pRight
|
| +*/
|
| +#ifdef SQLITE_DEBUG
|
| +static int fts5BufferCompareBlob(
|
| + Fts5Buffer *pLeft, /* Left hand side of comparison */
|
| + const u8 *pRight, int nRight /* Right hand side of comparison */
|
| +){
|
| + int nCmp = MIN(pLeft->n, nRight);
|
| + int res = memcmp(pLeft->p, pRight, nCmp);
|
| + return (res==0 ? (pLeft->n - nRight) : res);
|
| +}
|
| +#endif
|
| +
|
| +/*
|
| +** Compare the contents of the two buffers using memcmp(). If one buffer
|
| +** is a prefix of the other, it is considered the lesser.
|
| +**
|
| +** Return -ve if pLeft is smaller than pRight, 0 if they are equal or
|
| +** +ve if pRight is smaller than pLeft. In other words:
|
| +**
|
| +** res = *pLeft - *pRight
|
| +*/
|
| +static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){
|
| + int nCmp = MIN(pLeft->n, pRight->n);
|
| + int res = memcmp(pLeft->p, pRight->p, nCmp);
|
| + return (res==0 ? (pLeft->n - pRight->n) : res);
|
| +}
|
| +
|
| +#ifdef SQLITE_DEBUG
|
| +static int fts5BlobCompare(
|
| + const u8 *pLeft, int nLeft,
|
| + const u8 *pRight, int nRight
|
| +){
|
| + int nCmp = MIN(nLeft, nRight);
|
| + int res = memcmp(pLeft, pRight, nCmp);
|
| + return (res==0 ? (nLeft - nRight) : res);
|
| +}
|
| +#endif
|
| +
|
| +static int fts5LeafFirstTermOff(Fts5Data *pLeaf){
|
| + int ret;
|
| + fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret);
|
| + return ret;
|
| +}
|
| +
|
| +/*
|
| +** Close the read-only blob handle, if it is open.
|
| +*/
|
| +static void fts5CloseReader(Fts5Index *p){
|
| + if( p->pReader ){
|
| + sqlite3_blob *pReader = p->pReader;
|
| + p->pReader = 0;
|
| + sqlite3_blob_close(pReader);
|
| + }
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Retrieve a record from the %_data table.
|
| +**
|
| +** If an error occurs, NULL is returned and an error left in the
|
| +** Fts5Index object.
|
| +*/
|
| +static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
|
| + Fts5Data *pRet = 0;
|
| + if( p->rc==SQLITE_OK ){
|
| + int rc = SQLITE_OK;
|
| +
|
| + if( p->pReader ){
|
| + /* This call may return SQLITE_ABORT if there has been a savepoint
|
| + ** rollback since it was last used. In this case a new blob handle
|
| + ** is required. */
|
| + sqlite3_blob *pBlob = p->pReader;
|
| + p->pReader = 0;
|
| + rc = sqlite3_blob_reopen(pBlob, iRowid);
|
| + assert( p->pReader==0 );
|
| + p->pReader = pBlob;
|
| + if( rc!=SQLITE_OK ){
|
| + fts5CloseReader(p);
|
| + }
|
| + if( rc==SQLITE_ABORT ) rc = SQLITE_OK;
|
| + }
|
| +
|
| + /* If the blob handle is not open at this point, open it and seek
|
| + ** to the requested entry. */
|
| + if( p->pReader==0 && rc==SQLITE_OK ){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + rc = sqlite3_blob_open(pConfig->db,
|
| + pConfig->zDb, p->zDataTbl, "block", iRowid, 0, &p->pReader
|
| + );
|
| + }
|
| +
|
| + /* If either of the sqlite3_blob_open() or sqlite3_blob_reopen() calls
|
| + ** above returned SQLITE_ERROR, return SQLITE_CORRUPT_VTAB instead.
|
| + ** All the reasons those functions might return SQLITE_ERROR - missing
|
| + ** table, missing row, non-blob/text in block column - indicate
|
| + ** backing store corruption. */
|
| + if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT;
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + u8 *aOut = 0; /* Read blob data into this buffer */
|
| + int nByte = sqlite3_blob_bytes(p->pReader);
|
| + int nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING;
|
| + pRet = (Fts5Data*)sqlite3_malloc(nAlloc);
|
| + if( pRet ){
|
| + pRet->nn = nByte;
|
| + aOut = pRet->p = (u8*)&pRet[1];
|
| + }else{
|
| + rc = SQLITE_NOMEM;
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3_blob_read(p->pReader, aOut, nByte, 0);
|
| + }
|
| + if( rc!=SQLITE_OK ){
|
| + sqlite3_free(pRet);
|
| + pRet = 0;
|
| + }else{
|
| + /* TODO1: Fix this */
|
| + pRet->szLeaf = fts5GetU16(&pRet->p[2]);
|
| + }
|
| + }
|
| + p->rc = rc;
|
| + p->nRead++;
|
| + }
|
| +
|
| + assert( (pRet==0)==(p->rc!=SQLITE_OK) );
|
| + return pRet;
|
| +}
|
| +
|
| +/*
|
| +** Release a reference to data record returned by an earlier call to
|
| +** fts5DataRead().
|
| +*/
|
| +static void fts5DataRelease(Fts5Data *pData){
|
| + sqlite3_free(pData);
|
| +}
|
| +
|
| +static int fts5IndexPrepareStmt(
|
| + Fts5Index *p,
|
| + sqlite3_stmt **ppStmt,
|
| + char *zSql
|
| +){
|
| + if( p->rc==SQLITE_OK ){
|
| + if( zSql ){
|
| + p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0);
|
| + }else{
|
| + p->rc = SQLITE_NOMEM;
|
| + }
|
| + }
|
| + sqlite3_free(zSql);
|
| + return p->rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** INSERT OR REPLACE a record into the %_data table.
|
| +*/
|
| +static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){
|
| + if( p->rc!=SQLITE_OK ) return;
|
| +
|
| + if( p->pWriter==0 ){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf(
|
| + "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)",
|
| + pConfig->zDb, pConfig->zName
|
| + ));
|
| + if( p->rc ) return;
|
| + }
|
| +
|
| + sqlite3_bind_int64(p->pWriter, 1, iRowid);
|
| + sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC);
|
| + sqlite3_step(p->pWriter);
|
| + p->rc = sqlite3_reset(p->pWriter);
|
| +}
|
| +
|
| +/*
|
| +** Execute the following SQL:
|
| +**
|
| +** DELETE FROM %_data WHERE id BETWEEN $iFirst AND $iLast
|
| +*/
|
| +static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){
|
| + if( p->rc!=SQLITE_OK ) return;
|
| +
|
| + if( p->pDeleter==0 ){
|
| + int rc;
|
| + Fts5Config *pConfig = p->pConfig;
|
| + char *zSql = sqlite3_mprintf(
|
| + "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?",
|
| + pConfig->zDb, pConfig->zName
|
| + );
|
| + if( zSql==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0);
|
| + sqlite3_free(zSql);
|
| + }
|
| + if( rc!=SQLITE_OK ){
|
| + p->rc = rc;
|
| + return;
|
| + }
|
| + }
|
| +
|
| + sqlite3_bind_int64(p->pDeleter, 1, iFirst);
|
| + sqlite3_bind_int64(p->pDeleter, 2, iLast);
|
| + sqlite3_step(p->pDeleter);
|
| + p->rc = sqlite3_reset(p->pDeleter);
|
| +}
|
| +
|
| +/*
|
| +** Remove all records associated with segment iSegid.
|
| +*/
|
| +static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){
|
| + i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0);
|
| + i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1;
|
| + fts5DataDelete(p, iFirst, iLast);
|
| + if( p->pIdxDeleter==0 ){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf(
|
| + "DELETE FROM '%q'.'%q_idx' WHERE segid=?",
|
| + pConfig->zDb, pConfig->zName
|
| + ));
|
| + }
|
| + if( p->rc==SQLITE_OK ){
|
| + sqlite3_bind_int(p->pIdxDeleter, 1, iSegid);
|
| + sqlite3_step(p->pIdxDeleter);
|
| + p->rc = sqlite3_reset(p->pIdxDeleter);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Release a reference to an Fts5Structure object returned by an earlier
|
| +** call to fts5StructureRead() or fts5StructureDecode().
|
| +*/
|
| +static void fts5StructureRelease(Fts5Structure *pStruct){
|
| + if( pStruct && 0>=(--pStruct->nRef) ){
|
| + int i;
|
| + assert( pStruct->nRef==0 );
|
| + for(i=0; i<pStruct->nLevel; i++){
|
| + sqlite3_free(pStruct->aLevel[i].aSeg);
|
| + }
|
| + sqlite3_free(pStruct);
|
| + }
|
| +}
|
| +
|
| +static void fts5StructureRef(Fts5Structure *pStruct){
|
| + pStruct->nRef++;
|
| +}
|
| +
|
| +/*
|
| +** Deserialize and return the structure record currently stored in serialized
|
| +** form within buffer pData/nData.
|
| +**
|
| +** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
|
| +** are over-allocated by one slot. This allows the structure contents
|
| +** to be more easily edited.
|
| +**
|
| +** If an error occurs, *ppOut is set to NULL and an SQLite error code
|
| +** returned. Otherwise, *ppOut is set to point to the new object and
|
| +** SQLITE_OK returned.
|
| +*/
|
| +static int fts5StructureDecode(
|
| + const u8 *pData, /* Buffer containing serialized structure */
|
| + int nData, /* Size of buffer pData in bytes */
|
| + int *piCookie, /* Configuration cookie value */
|
| + Fts5Structure **ppOut /* OUT: Deserialized object */
|
| +){
|
| + int rc = SQLITE_OK;
|
| + int i = 0;
|
| + int iLvl;
|
| + int nLevel = 0;
|
| + int nSegment = 0;
|
| + int nByte; /* Bytes of space to allocate at pRet */
|
| + Fts5Structure *pRet = 0; /* Structure object to return */
|
| +
|
| + /* Grab the cookie value */
|
| + if( piCookie ) *piCookie = sqlite3Fts5Get32(pData);
|
| + i = 4;
|
| +
|
| + /* Read the total number of levels and segments from the start of the
|
| + ** structure record. */
|
| + i += fts5GetVarint32(&pData[i], nLevel);
|
| + i += fts5GetVarint32(&pData[i], nSegment);
|
| + nByte = (
|
| + sizeof(Fts5Structure) + /* Main structure */
|
| + sizeof(Fts5StructureLevel) * (nLevel-1) /* aLevel[] array */
|
| + );
|
| + pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte);
|
| +
|
| + if( pRet ){
|
| + pRet->nRef = 1;
|
| + pRet->nLevel = nLevel;
|
| + pRet->nSegment = nSegment;
|
| + i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter);
|
| +
|
| + for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){
|
| + Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl];
|
| + int nTotal;
|
| + int iSeg;
|
| +
|
| + i += fts5GetVarint32(&pData[i], pLvl->nMerge);
|
| + i += fts5GetVarint32(&pData[i], nTotal);
|
| + assert( nTotal>=pLvl->nMerge );
|
| + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc,
|
| + nTotal * sizeof(Fts5StructureSegment)
|
| + );
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + pLvl->nSeg = nTotal;
|
| + for(iSeg=0; iSeg<nTotal; iSeg++){
|
| + i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid);
|
| + i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst);
|
| + i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast);
|
| + }
|
| + }else{
|
| + fts5StructureRelease(pRet);
|
| + pRet = 0;
|
| + }
|
| + }
|
| + }
|
| +
|
| + *ppOut = pRet;
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +**
|
| +*/
|
| +static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){
|
| + if( *pRc==SQLITE_OK ){
|
| + Fts5Structure *pStruct = *ppStruct;
|
| + int nLevel = pStruct->nLevel;
|
| + int nByte = (
|
| + sizeof(Fts5Structure) + /* Main structure */
|
| + sizeof(Fts5StructureLevel) * (nLevel+1) /* aLevel[] array */
|
| + );
|
| +
|
| + pStruct = sqlite3_realloc(pStruct, nByte);
|
| + if( pStruct ){
|
| + memset(&pStruct->aLevel[nLevel], 0, sizeof(Fts5StructureLevel));
|
| + pStruct->nLevel++;
|
| + *ppStruct = pStruct;
|
| + }else{
|
| + *pRc = SQLITE_NOMEM;
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Extend level iLvl so that there is room for at least nExtra more
|
| +** segments.
|
| +*/
|
| +static void fts5StructureExtendLevel(
|
| + int *pRc,
|
| + Fts5Structure *pStruct,
|
| + int iLvl,
|
| + int nExtra,
|
| + int bInsert
|
| +){
|
| + if( *pRc==SQLITE_OK ){
|
| + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
|
| + Fts5StructureSegment *aNew;
|
| + int nByte;
|
| +
|
| + nByte = (pLvl->nSeg + nExtra) * sizeof(Fts5StructureSegment);
|
| + aNew = sqlite3_realloc(pLvl->aSeg, nByte);
|
| + if( aNew ){
|
| + if( bInsert==0 ){
|
| + memset(&aNew[pLvl->nSeg], 0, sizeof(Fts5StructureSegment) * nExtra);
|
| + }else{
|
| + int nMove = pLvl->nSeg * sizeof(Fts5StructureSegment);
|
| + memmove(&aNew[nExtra], aNew, nMove);
|
| + memset(aNew, 0, sizeof(Fts5StructureSegment) * nExtra);
|
| + }
|
| + pLvl->aSeg = aNew;
|
| + }else{
|
| + *pRc = SQLITE_NOMEM;
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Read, deserialize and return the structure record.
|
| +**
|
| +** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
|
| +** are over-allocated as described for function fts5StructureDecode()
|
| +** above.
|
| +**
|
| +** If an error occurs, NULL is returned and an error code left in the
|
| +** Fts5Index handle. If an error has already occurred when this function
|
| +** is called, it is a no-op.
|
| +*/
|
| +static Fts5Structure *fts5StructureRead(Fts5Index *p){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + Fts5Structure *pRet = 0; /* Object to return */
|
| + int iCookie; /* Configuration cookie */
|
| + Fts5Data *pData;
|
| +
|
| + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
|
| + if( p->rc ) return 0;
|
| + /* TODO: Do we need this if the leaf-index is appended? Probably... */
|
| + memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
|
| + p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
|
| + if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
|
| + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
|
| + }
|
| +
|
| + fts5DataRelease(pData);
|
| + if( p->rc!=SQLITE_OK ){
|
| + fts5StructureRelease(pRet);
|
| + pRet = 0;
|
| + }
|
| + return pRet;
|
| +}
|
| +
|
| +/*
|
| +** Return the total number of segments in index structure pStruct. This
|
| +** function is only ever used as part of assert() conditions.
|
| +*/
|
| +#ifdef SQLITE_DEBUG
|
| +static int fts5StructureCountSegments(Fts5Structure *pStruct){
|
| + int nSegment = 0; /* Total number of segments */
|
| + if( pStruct ){
|
| + int iLvl; /* Used to iterate through levels */
|
| + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
|
| + nSegment += pStruct->aLevel[iLvl].nSeg;
|
| + }
|
| + }
|
| +
|
| + return nSegment;
|
| +}
|
| +#endif
|
| +
|
| +#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \
|
| + assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) ); \
|
| + memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \
|
| + (pBuf)->n += nBlob; \
|
| +}
|
| +
|
| +#define fts5BufferSafeAppendVarint(pBuf, iVal) { \
|
| + (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \
|
| + assert( (pBuf)->nSpace>=(pBuf)->n ); \
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Serialize and store the "structure" record.
|
| +**
|
| +** If an error occurs, leave an error code in the Fts5Index object. If an
|
| +** error has already occurred, this function is a no-op.
|
| +*/
|
| +static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
|
| + if( p->rc==SQLITE_OK ){
|
| + Fts5Buffer buf; /* Buffer to serialize record into */
|
| + int iLvl; /* Used to iterate through levels */
|
| + int iCookie; /* Cookie value to store */
|
| +
|
| + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
|
| + memset(&buf, 0, sizeof(Fts5Buffer));
|
| +
|
| + /* Append the current configuration cookie */
|
| + iCookie = p->pConfig->iCookie;
|
| + if( iCookie<0 ) iCookie = 0;
|
| +
|
| + if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){
|
| + sqlite3Fts5Put32(buf.p, iCookie);
|
| + buf.n = 4;
|
| + fts5BufferSafeAppendVarint(&buf, pStruct->nLevel);
|
| + fts5BufferSafeAppendVarint(&buf, pStruct->nSegment);
|
| + fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter);
|
| + }
|
| +
|
| + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
|
| + int iSeg; /* Used to iterate through segments */
|
| + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
|
| + fts5BufferAppendVarint(&p->rc, &buf, pLvl->nMerge);
|
| + fts5BufferAppendVarint(&p->rc, &buf, pLvl->nSeg);
|
| + assert( pLvl->nMerge<=pLvl->nSeg );
|
| +
|
| + for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
|
| + fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid);
|
| + fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst);
|
| + fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
|
| + }
|
| + }
|
| +
|
| + fts5DataWrite(p, FTS5_STRUCTURE_ROWID, buf.p, buf.n);
|
| + fts5BufferFree(&buf);
|
| + }
|
| +}
|
| +
|
| +#if 0
|
| +static void fts5DebugStructure(int*,Fts5Buffer*,Fts5Structure*);
|
| +static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){
|
| + int rc = SQLITE_OK;
|
| + Fts5Buffer buf;
|
| + memset(&buf, 0, sizeof(buf));
|
| + fts5DebugStructure(&rc, &buf, pStruct);
|
| + fprintf(stdout, "%s: %s\n", zCaption, buf.p);
|
| + fflush(stdout);
|
| + fts5BufferFree(&buf);
|
| +}
|
| +#else
|
| +# define fts5PrintStructure(x,y)
|
| +#endif
|
| +
|
| +static int fts5SegmentSize(Fts5StructureSegment *pSeg){
|
| + return 1 + pSeg->pgnoLast - pSeg->pgnoFirst;
|
| +}
|
| +
|
| +/*
|
| +** Return a copy of index structure pStruct. Except, promote as many
|
| +** segments as possible to level iPromote. If an OOM occurs, NULL is
|
| +** returned.
|
| +*/
|
| +static void fts5StructurePromoteTo(
|
| + Fts5Index *p,
|
| + int iPromote,
|
| + int szPromote,
|
| + Fts5Structure *pStruct
|
| +){
|
| + int il, is;
|
| + Fts5StructureLevel *pOut = &pStruct->aLevel[iPromote];
|
| +
|
| + if( pOut->nMerge==0 ){
|
| + for(il=iPromote+1; il<pStruct->nLevel; il++){
|
| + Fts5StructureLevel *pLvl = &pStruct->aLevel[il];
|
| + if( pLvl->nMerge ) return;
|
| + for(is=pLvl->nSeg-1; is>=0; is--){
|
| + int sz = fts5SegmentSize(&pLvl->aSeg[is]);
|
| + if( sz>szPromote ) return;
|
| + fts5StructureExtendLevel(&p->rc, pStruct, iPromote, 1, 1);
|
| + if( p->rc ) return;
|
| + memcpy(pOut->aSeg, &pLvl->aSeg[is], sizeof(Fts5StructureSegment));
|
| + pOut->nSeg++;
|
| + pLvl->nSeg--;
|
| + }
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** A new segment has just been written to level iLvl of index structure
|
| +** pStruct. This function determines if any segments should be promoted
|
| +** as a result. Segments are promoted in two scenarios:
|
| +**
|
| +** a) If the segment just written is smaller than one or more segments
|
| +** within the previous populated level, it is promoted to the previous
|
| +** populated level.
|
| +**
|
| +** b) If the segment just written is larger than the newest segment on
|
| +** the next populated level, then that segment, and any other adjacent
|
| +** segments that are also smaller than the one just written, are
|
| +** promoted.
|
| +**
|
| +** If one or more segments are promoted, the structure object is updated
|
| +** to reflect this.
|
| +*/
|
| +static void fts5StructurePromote(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + int iLvl, /* Index level just updated */
|
| + Fts5Structure *pStruct /* Index structure */
|
| +){
|
| + if( p->rc==SQLITE_OK ){
|
| + int iTst;
|
| + int iPromote = -1;
|
| + int szPromote = 0; /* Promote anything this size or smaller */
|
| + Fts5StructureSegment *pSeg; /* Segment just written */
|
| + int szSeg; /* Size of segment just written */
|
| + int nSeg = pStruct->aLevel[iLvl].nSeg;
|
| +
|
| + if( nSeg==0 ) return;
|
| + pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1];
|
| + szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst);
|
| +
|
| + /* Check for condition (a) */
|
| + for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--);
|
| + if( iTst>=0 ){
|
| + int i;
|
| + int szMax = 0;
|
| + Fts5StructureLevel *pTst = &pStruct->aLevel[iTst];
|
| + assert( pTst->nMerge==0 );
|
| + for(i=0; i<pTst->nSeg; i++){
|
| + int sz = pTst->aSeg[i].pgnoLast - pTst->aSeg[i].pgnoFirst + 1;
|
| + if( sz>szMax ) szMax = sz;
|
| + }
|
| + if( szMax>=szSeg ){
|
| + /* Condition (a) is true. Promote the newest segment on level
|
| + ** iLvl to level iTst. */
|
| + iPromote = iTst;
|
| + szPromote = szMax;
|
| + }
|
| + }
|
| +
|
| + /* If condition (a) is not met, assume (b) is true. StructurePromoteTo()
|
| + ** is a no-op if it is not. */
|
| + if( iPromote<0 ){
|
| + iPromote = iLvl;
|
| + szPromote = szSeg;
|
| + }
|
| + fts5StructurePromoteTo(p, iPromote, szPromote, pStruct);
|
| + }
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Advance the iterator passed as the only argument. If the end of the
|
| +** doclist-index page is reached, return non-zero.
|
| +*/
|
| +static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){
|
| + Fts5Data *pData = pLvl->pData;
|
| +
|
| + if( pLvl->iOff==0 ){
|
| + assert( pLvl->bEof==0 );
|
| + pLvl->iOff = 1;
|
| + pLvl->iOff += fts5GetVarint32(&pData->p[1], pLvl->iLeafPgno);
|
| + pLvl->iOff += fts5GetVarint(&pData->p[pLvl->iOff], (u64*)&pLvl->iRowid);
|
| + pLvl->iFirstOff = pLvl->iOff;
|
| + }else{
|
| + int iOff;
|
| + for(iOff=pLvl->iOff; iOff<pData->nn; iOff++){
|
| + if( pData->p[iOff] ) break;
|
| + }
|
| +
|
| + if( iOff<pData->nn ){
|
| + i64 iVal;
|
| + pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1;
|
| + iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal);
|
| + pLvl->iRowid += iVal;
|
| + pLvl->iOff = iOff;
|
| + }else{
|
| + pLvl->bEof = 1;
|
| + }
|
| + }
|
| +
|
| + return pLvl->bEof;
|
| +}
|
| +
|
| +/*
|
| +** Advance the iterator passed as the only argument.
|
| +*/
|
| +static int fts5DlidxIterNextR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){
|
| + Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl];
|
| +
|
| + assert( iLvl<pIter->nLvl );
|
| + if( fts5DlidxLvlNext(pLvl) ){
|
| + if( (iLvl+1) < pIter->nLvl ){
|
| + fts5DlidxIterNextR(p, pIter, iLvl+1);
|
| + if( pLvl[1].bEof==0 ){
|
| + fts5DataRelease(pLvl->pData);
|
| + memset(pLvl, 0, sizeof(Fts5DlidxLvl));
|
| + pLvl->pData = fts5DataRead(p,
|
| + FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno)
|
| + );
|
| + if( pLvl->pData ) fts5DlidxLvlNext(pLvl);
|
| + }
|
| + }
|
| + }
|
| +
|
| + return pIter->aLvl[0].bEof;
|
| +}
|
| +static int fts5DlidxIterNext(Fts5Index *p, Fts5DlidxIter *pIter){
|
| + return fts5DlidxIterNextR(p, pIter, 0);
|
| +}
|
| +
|
| +/*
|
| +** The iterator passed as the first argument has the following fields set
|
| +** as follows. This function sets up the rest of the iterator so that it
|
| +** points to the first rowid in the doclist-index.
|
| +**
|
| +** pData:
|
| +** pointer to doclist-index record,
|
| +**
|
| +** When this function is called pIter->iLeafPgno is the page number the
|
| +** doclist is associated with (the one featuring the term).
|
| +*/
|
| +static int fts5DlidxIterFirst(Fts5DlidxIter *pIter){
|
| + int i;
|
| + for(i=0; i<pIter->nLvl; i++){
|
| + fts5DlidxLvlNext(&pIter->aLvl[i]);
|
| + }
|
| + return pIter->aLvl[0].bEof;
|
| +}
|
| +
|
| +
|
| +static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){
|
| + return p->rc!=SQLITE_OK || pIter->aLvl[0].bEof;
|
| +}
|
| +
|
| +static void fts5DlidxIterLast(Fts5Index *p, Fts5DlidxIter *pIter){
|
| + int i;
|
| +
|
| + /* Advance each level to the last entry on the last page */
|
| + for(i=pIter->nLvl-1; p->rc==SQLITE_OK && i>=0; i--){
|
| + Fts5DlidxLvl *pLvl = &pIter->aLvl[i];
|
| + while( fts5DlidxLvlNext(pLvl)==0 );
|
| + pLvl->bEof = 0;
|
| +
|
| + if( i>0 ){
|
| + Fts5DlidxLvl *pChild = &pLvl[-1];
|
| + fts5DataRelease(pChild->pData);
|
| + memset(pChild, 0, sizeof(Fts5DlidxLvl));
|
| + pChild->pData = fts5DataRead(p,
|
| + FTS5_DLIDX_ROWID(pIter->iSegid, i-1, pLvl->iLeafPgno)
|
| + );
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Move the iterator passed as the only argument to the previous entry.
|
| +*/
|
| +static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){
|
| + int iOff = pLvl->iOff;
|
| +
|
| + assert( pLvl->bEof==0 );
|
| + if( iOff<=pLvl->iFirstOff ){
|
| + pLvl->bEof = 1;
|
| + }else{
|
| + u8 *a = pLvl->pData->p;
|
| + i64 iVal;
|
| + int iLimit;
|
| + int ii;
|
| + int nZero = 0;
|
| +
|
| + /* Currently iOff points to the first byte of a varint. This block
|
| + ** decrements iOff until it points to the first byte of the previous
|
| + ** varint. Taking care not to read any memory locations that occur
|
| + ** before the buffer in memory. */
|
| + iLimit = (iOff>9 ? iOff-9 : 0);
|
| + for(iOff--; iOff>iLimit; iOff--){
|
| + if( (a[iOff-1] & 0x80)==0 ) break;
|
| + }
|
| +
|
| + fts5GetVarint(&a[iOff], (u64*)&iVal);
|
| + pLvl->iRowid -= iVal;
|
| + pLvl->iLeafPgno--;
|
| +
|
| + /* Skip backwards past any 0x00 varints. */
|
| + for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){
|
| + nZero++;
|
| + }
|
| + if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){
|
| + /* The byte immediately before the last 0x00 byte has the 0x80 bit
|
| + ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80
|
| + ** bytes before a[ii]. */
|
| + int bZero = 0; /* True if last 0x00 counts */
|
| + if( (ii-8)>=pLvl->iFirstOff ){
|
| + int j;
|
| + for(j=1; j<=8 && (a[ii-j] & 0x80); j++);
|
| + bZero = (j>8);
|
| + }
|
| + if( bZero==0 ) nZero--;
|
| + }
|
| + pLvl->iLeafPgno -= nZero;
|
| + pLvl->iOff = iOff - nZero;
|
| + }
|
| +
|
| + return pLvl->bEof;
|
| +}
|
| +
|
| +static int fts5DlidxIterPrevR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){
|
| + Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl];
|
| +
|
| + assert( iLvl<pIter->nLvl );
|
| + if( fts5DlidxLvlPrev(pLvl) ){
|
| + if( (iLvl+1) < pIter->nLvl ){
|
| + fts5DlidxIterPrevR(p, pIter, iLvl+1);
|
| + if( pLvl[1].bEof==0 ){
|
| + fts5DataRelease(pLvl->pData);
|
| + memset(pLvl, 0, sizeof(Fts5DlidxLvl));
|
| + pLvl->pData = fts5DataRead(p,
|
| + FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno)
|
| + );
|
| + if( pLvl->pData ){
|
| + while( fts5DlidxLvlNext(pLvl)==0 );
|
| + pLvl->bEof = 0;
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + return pIter->aLvl[0].bEof;
|
| +}
|
| +static int fts5DlidxIterPrev(Fts5Index *p, Fts5DlidxIter *pIter){
|
| + return fts5DlidxIterPrevR(p, pIter, 0);
|
| +}
|
| +
|
| +/*
|
| +** Free a doclist-index iterator object allocated by fts5DlidxIterInit().
|
| +*/
|
| +static void fts5DlidxIterFree(Fts5DlidxIter *pIter){
|
| + if( pIter ){
|
| + int i;
|
| + for(i=0; i<pIter->nLvl; i++){
|
| + fts5DataRelease(pIter->aLvl[i].pData);
|
| + }
|
| + sqlite3_free(pIter);
|
| + }
|
| +}
|
| +
|
| +static Fts5DlidxIter *fts5DlidxIterInit(
|
| + Fts5Index *p, /* Fts5 Backend to iterate within */
|
| + int bRev, /* True for ORDER BY ASC */
|
| + int iSegid, /* Segment id */
|
| + int iLeafPg /* Leaf page number to load dlidx for */
|
| +){
|
| + Fts5DlidxIter *pIter = 0;
|
| + int i;
|
| + int bDone = 0;
|
| +
|
| + for(i=0; p->rc==SQLITE_OK && bDone==0; i++){
|
| + int nByte = sizeof(Fts5DlidxIter) + i * sizeof(Fts5DlidxLvl);
|
| + Fts5DlidxIter *pNew;
|
| +
|
| + pNew = (Fts5DlidxIter*)sqlite3_realloc(pIter, nByte);
|
| + if( pNew==0 ){
|
| + p->rc = SQLITE_NOMEM;
|
| + }else{
|
| + i64 iRowid = FTS5_DLIDX_ROWID(iSegid, i, iLeafPg);
|
| + Fts5DlidxLvl *pLvl = &pNew->aLvl[i];
|
| + pIter = pNew;
|
| + memset(pLvl, 0, sizeof(Fts5DlidxLvl));
|
| + pLvl->pData = fts5DataRead(p, iRowid);
|
| + if( pLvl->pData && (pLvl->pData->p[0] & 0x0001)==0 ){
|
| + bDone = 1;
|
| + }
|
| + pIter->nLvl = i+1;
|
| + }
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + pIter->iSegid = iSegid;
|
| + if( bRev==0 ){
|
| + fts5DlidxIterFirst(pIter);
|
| + }else{
|
| + fts5DlidxIterLast(p, pIter);
|
| + }
|
| + }
|
| +
|
| + if( p->rc!=SQLITE_OK ){
|
| + fts5DlidxIterFree(pIter);
|
| + pIter = 0;
|
| + }
|
| +
|
| + return pIter;
|
| +}
|
| +
|
| +static i64 fts5DlidxIterRowid(Fts5DlidxIter *pIter){
|
| + return pIter->aLvl[0].iRowid;
|
| +}
|
| +static int fts5DlidxIterPgno(Fts5DlidxIter *pIter){
|
| + return pIter->aLvl[0].iLeafPgno;
|
| +}
|
| +
|
| +/*
|
| +** Load the next leaf page into the segment iterator.
|
| +*/
|
| +static void fts5SegIterNextPage(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5SegIter *pIter /* Iterator to advance to next page */
|
| +){
|
| + Fts5Data *pLeaf;
|
| + Fts5StructureSegment *pSeg = pIter->pSeg;
|
| + fts5DataRelease(pIter->pLeaf);
|
| + pIter->iLeafPgno++;
|
| + if( pIter->pNextLeaf ){
|
| + pIter->pLeaf = pIter->pNextLeaf;
|
| + pIter->pNextLeaf = 0;
|
| + }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
|
| + pIter->pLeaf = fts5DataRead(p,
|
| + FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno)
|
| + );
|
| + }else{
|
| + pIter->pLeaf = 0;
|
| + }
|
| + pLeaf = pIter->pLeaf;
|
| +
|
| + if( pLeaf ){
|
| + pIter->iPgidxOff = pLeaf->szLeaf;
|
| + if( fts5LeafIsTermless(pLeaf) ){
|
| + pIter->iEndofDoclist = pLeaf->nn+1;
|
| + }else{
|
| + pIter->iPgidxOff += fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff],
|
| + pIter->iEndofDoclist
|
| + );
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Argument p points to a buffer containing a varint to be interpreted as a
|
| +** position list size field. Read the varint and return the number of bytes
|
| +** read. Before returning, set *pnSz to the number of bytes in the position
|
| +** list, and *pbDel to true if the delete flag is set, or false otherwise.
|
| +*/
|
| +static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){
|
| + int nSz;
|
| + int n = 0;
|
| + fts5FastGetVarint32(p, n, nSz);
|
| + assert_nc( nSz>=0 );
|
| + *pnSz = nSz/2;
|
| + *pbDel = nSz & 0x0001;
|
| + return n;
|
| +}
|
| +
|
| +/*
|
| +** Fts5SegIter.iLeafOffset currently points to the first byte of a
|
| +** position-list size field. Read the value of the field and store it
|
| +** in the following variables:
|
| +**
|
| +** Fts5SegIter.nPos
|
| +** Fts5SegIter.bDel
|
| +**
|
| +** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the
|
| +** position list content (if any).
|
| +*/
|
| +static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
|
| + if( p->rc==SQLITE_OK ){
|
| + int iOff = pIter->iLeafOffset; /* Offset to read at */
|
| + int nSz;
|
| + ASSERT_SZLEAF_OK(pIter->pLeaf);
|
| + fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz);
|
| + pIter->bDel = (nSz & 0x0001);
|
| + pIter->nPos = nSz>>1;
|
| + pIter->iLeafOffset = iOff;
|
| + assert_nc( pIter->nPos>=0 );
|
| + }
|
| +}
|
| +
|
| +static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
|
| + u8 *a = pIter->pLeaf->p; /* Buffer to read data from */
|
| + int iOff = pIter->iLeafOffset;
|
| +
|
| + ASSERT_SZLEAF_OK(pIter->pLeaf);
|
| + if( iOff>=pIter->pLeaf->szLeaf ){
|
| + fts5SegIterNextPage(p, pIter);
|
| + if( pIter->pLeaf==0 ){
|
| + if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT;
|
| + return;
|
| + }
|
| + iOff = 4;
|
| + a = pIter->pLeaf->p;
|
| + }
|
| + iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
|
| + pIter->iLeafOffset = iOff;
|
| +}
|
| +
|
| +/*
|
| +** Fts5SegIter.iLeafOffset currently points to the first byte of the
|
| +** "nSuffix" field of a term. Function parameter nKeep contains the value
|
| +** of the "nPrefix" field (if there was one - it is passed 0 if this is
|
| +** the first term in the segment).
|
| +**
|
| +** This function populates:
|
| +**
|
| +** Fts5SegIter.term
|
| +** Fts5SegIter.rowid
|
| +**
|
| +** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the content of
|
| +** the first position list. The position list belonging to document
|
| +** (Fts5SegIter.iRowid).
|
| +*/
|
| +static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){
|
| + u8 *a = pIter->pLeaf->p; /* Buffer to read data from */
|
| + int iOff = pIter->iLeafOffset; /* Offset to read at */
|
| + int nNew; /* Bytes of new data */
|
| +
|
| + iOff += fts5GetVarint32(&a[iOff], nNew);
|
| + pIter->term.n = nKeep;
|
| + fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]);
|
| + iOff += nNew;
|
| + pIter->iTermLeafOffset = iOff;
|
| + pIter->iTermLeafPgno = pIter->iLeafPgno;
|
| + pIter->iLeafOffset = iOff;
|
| +
|
| + if( pIter->iPgidxOff>=pIter->pLeaf->nn ){
|
| + pIter->iEndofDoclist = pIter->pLeaf->nn+1;
|
| + }else{
|
| + int nExtra;
|
| + pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra);
|
| + pIter->iEndofDoclist += nExtra;
|
| + }
|
| +
|
| + fts5SegIterLoadRowid(p, pIter);
|
| +}
|
| +
|
| +/*
|
| +** Initialize the iterator object pIter to iterate through the entries in
|
| +** segment pSeg. The iterator is left pointing to the first entry when
|
| +** this function returns.
|
| +**
|
| +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If
|
| +** an error has already occurred when this function is called, it is a no-op.
|
| +*/
|
| +static void fts5SegIterInit(
|
| + Fts5Index *p, /* FTS index object */
|
| + Fts5StructureSegment *pSeg, /* Description of segment */
|
| + Fts5SegIter *pIter /* Object to populate */
|
| +){
|
| + if( pSeg->pgnoFirst==0 ){
|
| + /* This happens if the segment is being used as an input to an incremental
|
| + ** merge and all data has already been "trimmed". See function
|
| + ** fts5TrimSegments() for details. In this case leave the iterator empty.
|
| + ** The caller will see the (pIter->pLeaf==0) and assume the iterator is
|
| + ** at EOF already. */
|
| + assert( pIter->pLeaf==0 );
|
| + return;
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + memset(pIter, 0, sizeof(*pIter));
|
| + pIter->pSeg = pSeg;
|
| + pIter->iLeafPgno = pSeg->pgnoFirst-1;
|
| + fts5SegIterNextPage(p, pIter);
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + pIter->iLeafOffset = 4;
|
| + assert_nc( pIter->pLeaf->nn>4 );
|
| + assert( fts5LeafFirstTermOff(pIter->pLeaf)==4 );
|
| + pIter->iPgidxOff = pIter->pLeaf->szLeaf+1;
|
| + fts5SegIterLoadTerm(p, pIter, 0);
|
| + fts5SegIterLoadNPos(p, pIter);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** This function is only ever called on iterators created by calls to
|
| +** Fts5IndexQuery() with the FTS5INDEX_QUERY_DESC flag set.
|
| +**
|
| +** The iterator is in an unusual state when this function is called: the
|
| +** Fts5SegIter.iLeafOffset variable is set to the offset of the start of
|
| +** the position-list size field for the first relevant rowid on the page.
|
| +** Fts5SegIter.rowid is set, but nPos and bDel are not.
|
| +**
|
| +** This function advances the iterator so that it points to the last
|
| +** relevant rowid on the page and, if necessary, initializes the
|
| +** aRowidOffset[] and iRowidOffset variables. At this point the iterator
|
| +** is in its regular state - Fts5SegIter.iLeafOffset points to the first
|
| +** byte of the position list content associated with said rowid.
|
| +*/
|
| +static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){
|
| + int n = pIter->pLeaf->szLeaf;
|
| + int i = pIter->iLeafOffset;
|
| + u8 *a = pIter->pLeaf->p;
|
| + int iRowidOffset = 0;
|
| +
|
| + if( n>pIter->iEndofDoclist ){
|
| + n = pIter->iEndofDoclist;
|
| + }
|
| +
|
| + ASSERT_SZLEAF_OK(pIter->pLeaf);
|
| + while( 1 ){
|
| + i64 iDelta = 0;
|
| + int nPos;
|
| + int bDummy;
|
| +
|
| + i += fts5GetPoslistSize(&a[i], &nPos, &bDummy);
|
| + i += nPos;
|
| + if( i>=n ) break;
|
| + i += fts5GetVarint(&a[i], (u64*)&iDelta);
|
| + pIter->iRowid += iDelta;
|
| +
|
| + if( iRowidOffset>=pIter->nRowidOffset ){
|
| + int nNew = pIter->nRowidOffset + 8;
|
| + int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int));
|
| + if( aNew==0 ){
|
| + p->rc = SQLITE_NOMEM;
|
| + break;
|
| + }
|
| + pIter->aRowidOffset = aNew;
|
| + pIter->nRowidOffset = nNew;
|
| + }
|
| +
|
| + pIter->aRowidOffset[iRowidOffset++] = pIter->iLeafOffset;
|
| + pIter->iLeafOffset = i;
|
| + }
|
| + pIter->iRowidOffset = iRowidOffset;
|
| + fts5SegIterLoadNPos(p, pIter);
|
| +}
|
| +
|
| +/*
|
| +**
|
| +*/
|
| +static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
|
| + assert( pIter->flags & FTS5_SEGITER_REVERSE );
|
| + assert( pIter->flags & FTS5_SEGITER_ONETERM );
|
| +
|
| + fts5DataRelease(pIter->pLeaf);
|
| + pIter->pLeaf = 0;
|
| + while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){
|
| + Fts5Data *pNew;
|
| + pIter->iLeafPgno--;
|
| + pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
|
| + pIter->pSeg->iSegid, pIter->iLeafPgno
|
| + ));
|
| + if( pNew ){
|
| + /* iTermLeafOffset may be equal to szLeaf if the term is the last
|
| + ** thing on the page - i.e. the first rowid is on the following page.
|
| + ** In this case leave pIter->pLeaf==0, this iterator is at EOF. */
|
| + if( pIter->iLeafPgno==pIter->iTermLeafPgno ){
|
| + assert( pIter->pLeaf==0 );
|
| + if( pIter->iTermLeafOffset<pNew->szLeaf ){
|
| + pIter->pLeaf = pNew;
|
| + pIter->iLeafOffset = pIter->iTermLeafOffset;
|
| + }
|
| + }else{
|
| + int iRowidOff;
|
| + iRowidOff = fts5LeafFirstRowidOff(pNew);
|
| + if( iRowidOff ){
|
| + pIter->pLeaf = pNew;
|
| + pIter->iLeafOffset = iRowidOff;
|
| + }
|
| + }
|
| +
|
| + if( pIter->pLeaf ){
|
| + u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset];
|
| + pIter->iLeafOffset += fts5GetVarint(a, (u64*)&pIter->iRowid);
|
| + break;
|
| + }else{
|
| + fts5DataRelease(pNew);
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( pIter->pLeaf ){
|
| + pIter->iEndofDoclist = pIter->pLeaf->nn+1;
|
| + fts5SegIterReverseInitPage(p, pIter);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Return true if the iterator passed as the second argument currently
|
| +** points to a delete marker. A delete marker is an entry with a 0 byte
|
| +** position-list.
|
| +*/
|
| +static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){
|
| + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
|
| + return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0);
|
| +}
|
| +
|
| +/*
|
| +** Advance iterator pIter to the next entry.
|
| +**
|
| +** If an error occurs, Fts5Index.rc is set to an appropriate error code. It
|
| +** is not considered an error if the iterator reaches EOF. If an error has
|
| +** already occurred when this function is called, it is a no-op.
|
| +*/
|
| +static void fts5SegIterNext(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5SegIter *pIter, /* Iterator to advance */
|
| + int *pbNewTerm /* OUT: Set for new term */
|
| +){
|
| + assert( pbNewTerm==0 || *pbNewTerm==0 );
|
| + if( p->rc==SQLITE_OK ){
|
| + if( pIter->flags & FTS5_SEGITER_REVERSE ){
|
| + assert( pIter->pNextLeaf==0 );
|
| + if( pIter->iRowidOffset>0 ){
|
| + u8 *a = pIter->pLeaf->p;
|
| + int iOff;
|
| + int nPos;
|
| + int bDummy;
|
| + i64 iDelta;
|
| +
|
| + pIter->iRowidOffset--;
|
| + pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];
|
| + iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy);
|
| + iOff += nPos;
|
| + fts5GetVarint(&a[iOff], (u64*)&iDelta);
|
| + pIter->iRowid -= iDelta;
|
| + fts5SegIterLoadNPos(p, pIter);
|
| + }else{
|
| + fts5SegIterReverseNewPage(p, pIter);
|
| + }
|
| + }else{
|
| + Fts5Data *pLeaf = pIter->pLeaf;
|
| + int iOff;
|
| + int bNewTerm = 0;
|
| + int nKeep = 0;
|
| +
|
| + /* Search for the end of the position list within the current page. */
|
| + u8 *a = pLeaf->p;
|
| + int n = pLeaf->szLeaf;
|
| +
|
| + ASSERT_SZLEAF_OK(pLeaf);
|
| + iOff = pIter->iLeafOffset + pIter->nPos;
|
| +
|
| + if( iOff<n ){
|
| + /* The next entry is on the current page. */
|
| + assert_nc( iOff<=pIter->iEndofDoclist );
|
| + if( iOff>=pIter->iEndofDoclist ){
|
| + bNewTerm = 1;
|
| + if( iOff!=fts5LeafFirstTermOff(pLeaf) ){
|
| + iOff += fts5GetVarint32(&a[iOff], nKeep);
|
| + }
|
| + }else{
|
| + u64 iDelta;
|
| + iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta);
|
| + pIter->iRowid += iDelta;
|
| + assert_nc( iDelta>0 );
|
| + }
|
| + pIter->iLeafOffset = iOff;
|
| +
|
| + }else if( pIter->pSeg==0 ){
|
| + const u8 *pList = 0;
|
| + const char *zTerm = 0;
|
| + int nList = 0;
|
| + assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm );
|
| + if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){
|
| + sqlite3Fts5HashScanNext(p->pHash);
|
| + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList);
|
| + }
|
| + if( pList==0 ){
|
| + fts5DataRelease(pIter->pLeaf);
|
| + pIter->pLeaf = 0;
|
| + }else{
|
| + pIter->pLeaf->p = (u8*)pList;
|
| + pIter->pLeaf->nn = nList;
|
| + pIter->pLeaf->szLeaf = nList;
|
| + pIter->iEndofDoclist = nList+1;
|
| + sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm),
|
| + (u8*)zTerm);
|
| + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid);
|
| + *pbNewTerm = 1;
|
| + }
|
| + }else{
|
| + iOff = 0;
|
| + /* Next entry is not on the current page */
|
| + while( iOff==0 ){
|
| + fts5SegIterNextPage(p, pIter);
|
| + pLeaf = pIter->pLeaf;
|
| + if( pLeaf==0 ) break;
|
| + ASSERT_SZLEAF_OK(pLeaf);
|
| + if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){
|
| + iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
|
| + pIter->iLeafOffset = iOff;
|
| +
|
| + if( pLeaf->nn>pLeaf->szLeaf ){
|
| + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
|
| + &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
|
| + );
|
| + }
|
| +
|
| + }
|
| + else if( pLeaf->nn>pLeaf->szLeaf ){
|
| + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
|
| + &pLeaf->p[pLeaf->szLeaf], iOff
|
| + );
|
| + pIter->iLeafOffset = iOff;
|
| + pIter->iEndofDoclist = iOff;
|
| + bNewTerm = 1;
|
| + }
|
| + if( iOff>=pLeaf->szLeaf ){
|
| + p->rc = FTS5_CORRUPT;
|
| + return;
|
| + }
|
| + }
|
| + }
|
| +
|
| + /* Check if the iterator is now at EOF. If so, return early. */
|
| + if( pIter->pLeaf ){
|
| + if( bNewTerm ){
|
| + if( pIter->flags & FTS5_SEGITER_ONETERM ){
|
| + fts5DataRelease(pIter->pLeaf);
|
| + pIter->pLeaf = 0;
|
| + }else{
|
| + fts5SegIterLoadTerm(p, pIter, nKeep);
|
| + fts5SegIterLoadNPos(p, pIter);
|
| + if( pbNewTerm ) *pbNewTerm = 1;
|
| + }
|
| + }else{
|
| + /* The following could be done by calling fts5SegIterLoadNPos(). But
|
| + ** this block is particularly performance critical, so equivalent
|
| + ** code is inlined. */
|
| + int nSz;
|
| + assert( p->rc==SQLITE_OK );
|
| + fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
|
| + pIter->bDel = (nSz & 0x0001);
|
| + pIter->nPos = nSz>>1;
|
| + assert_nc( pIter->nPos>=0 );
|
| + }
|
| + }
|
| + }
|
| + }
|
| +}
|
| +
|
| +#define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; }
|
| +
|
| +/*
|
| +** Iterator pIter currently points to the first rowid in a doclist. This
|
| +** function sets the iterator up so that iterates in reverse order through
|
| +** the doclist.
|
| +*/
|
| +static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
|
| + Fts5DlidxIter *pDlidx = pIter->pDlidx;
|
| + Fts5Data *pLast = 0;
|
| + int pgnoLast = 0;
|
| +
|
| + if( pDlidx ){
|
| + int iSegid = pIter->pSeg->iSegid;
|
| + pgnoLast = fts5DlidxIterPgno(pDlidx);
|
| + pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast));
|
| + }else{
|
| + Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */
|
| +
|
| + /* Currently, Fts5SegIter.iLeafOffset points to the first byte of
|
| + ** position-list content for the current rowid. Back it up so that it
|
| + ** points to the start of the position-list size field. */
|
| + pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel);
|
| +
|
| + /* If this condition is true then the largest rowid for the current
|
| + ** term may not be stored on the current page. So search forward to
|
| + ** see where said rowid really is. */
|
| + if( pIter->iEndofDoclist>=pLeaf->szLeaf ){
|
| + int pgno;
|
| + Fts5StructureSegment *pSeg = pIter->pSeg;
|
| +
|
| + /* The last rowid in the doclist may not be on the current page. Search
|
| + ** forward to find the page containing the last rowid. */
|
| + for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){
|
| + i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno);
|
| + Fts5Data *pNew = fts5DataRead(p, iAbs);
|
| + if( pNew ){
|
| + int iRowid, bTermless;
|
| + iRowid = fts5LeafFirstRowidOff(pNew);
|
| + bTermless = fts5LeafIsTermless(pNew);
|
| + if( iRowid ){
|
| + SWAPVAL(Fts5Data*, pNew, pLast);
|
| + pgnoLast = pgno;
|
| + }
|
| + fts5DataRelease(pNew);
|
| + if( bTermless==0 ) break;
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + /* If pLast is NULL at this point, then the last rowid for this doclist
|
| + ** lies on the page currently indicated by the iterator. In this case
|
| + ** pIter->iLeafOffset is already set to point to the position-list size
|
| + ** field associated with the first relevant rowid on the page.
|
| + **
|
| + ** Or, if pLast is non-NULL, then it is the page that contains the last
|
| + ** rowid. In this case configure the iterator so that it points to the
|
| + ** first rowid on this page.
|
| + */
|
| + if( pLast ){
|
| + int iOff;
|
| + fts5DataRelease(pIter->pLeaf);
|
| + pIter->pLeaf = pLast;
|
| + pIter->iLeafPgno = pgnoLast;
|
| + iOff = fts5LeafFirstRowidOff(pLast);
|
| + iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid);
|
| + pIter->iLeafOffset = iOff;
|
| +
|
| + if( fts5LeafIsTermless(pLast) ){
|
| + pIter->iEndofDoclist = pLast->nn+1;
|
| + }else{
|
| + pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast);
|
| + }
|
| +
|
| + }
|
| +
|
| + fts5SegIterReverseInitPage(p, pIter);
|
| +}
|
| +
|
| +/*
|
| +** Iterator pIter currently points to the first rowid of a doclist.
|
| +** There is a doclist-index associated with the final term on the current
|
| +** page. If the current term is the last term on the page, load the
|
| +** doclist-index from disk and initialize an iterator at (pIter->pDlidx).
|
| +*/
|
| +static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){
|
| + int iSeg = pIter->pSeg->iSegid;
|
| + int bRev = (pIter->flags & FTS5_SEGITER_REVERSE);
|
| + Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */
|
| +
|
| + assert( pIter->flags & FTS5_SEGITER_ONETERM );
|
| + assert( pIter->pDlidx==0 );
|
| +
|
| + /* Check if the current doclist ends on this page. If it does, return
|
| + ** early without loading the doclist-index (as it belongs to a different
|
| + ** term. */
|
| + if( pIter->iTermLeafPgno==pIter->iLeafPgno
|
| + && pIter->iEndofDoclist<pLeaf->szLeaf
|
| + ){
|
| + return;
|
| + }
|
| +
|
| + pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
|
| +}
|
| +
|
| +#define fts5IndexSkipVarint(a, iOff) { \
|
| + int iEnd = iOff+9; \
|
| + while( (a[iOff++] & 0x80) && iOff<iEnd ); \
|
| +}
|
| +
|
| +/*
|
| +** The iterator object passed as the second argument currently contains
|
| +** no valid values except for the Fts5SegIter.pLeaf member variable. This
|
| +** function searches the leaf page for a term matching (pTerm/nTerm).
|
| +**
|
| +** If the specified term is found on the page, then the iterator is left
|
| +** pointing to it. If argument bGe is zero and the term is not found,
|
| +** the iterator is left pointing at EOF.
|
| +**
|
| +** If bGe is non-zero and the specified term is not found, then the
|
| +** iterator is left pointing to the smallest term in the segment that
|
| +** is larger than the specified term, even if this term is not on the
|
| +** current page.
|
| +*/
|
| +static void fts5LeafSeek(
|
| + Fts5Index *p, /* Leave any error code here */
|
| + int bGe, /* True for a >= search */
|
| + Fts5SegIter *pIter, /* Iterator to seek */
|
| + const u8 *pTerm, int nTerm /* Term to search for */
|
| +){
|
| + int iOff;
|
| + const u8 *a = pIter->pLeaf->p;
|
| + int szLeaf = pIter->pLeaf->szLeaf;
|
| + int n = pIter->pLeaf->nn;
|
| +
|
| + int nMatch = 0;
|
| + int nKeep = 0;
|
| + int nNew = 0;
|
| + int iTermOff;
|
| + int iPgidx; /* Current offset in pgidx */
|
| + int bEndOfPage = 0;
|
| +
|
| + assert( p->rc==SQLITE_OK );
|
| +
|
| + iPgidx = szLeaf;
|
| + iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff);
|
| + iOff = iTermOff;
|
| +
|
| + while( 1 ){
|
| +
|
| + /* Figure out how many new bytes are in this term */
|
| + fts5FastGetVarint32(a, iOff, nNew);
|
| + if( nKeep<nMatch ){
|
| + goto search_failed;
|
| + }
|
| +
|
| + assert( nKeep>=nMatch );
|
| + if( nKeep==nMatch ){
|
| + int nCmp;
|
| + int i;
|
| + nCmp = MIN(nNew, nTerm-nMatch);
|
| + for(i=0; i<nCmp; i++){
|
| + if( a[iOff+i]!=pTerm[nMatch+i] ) break;
|
| + }
|
| + nMatch += i;
|
| +
|
| + if( nTerm==nMatch ){
|
| + if( i==nNew ){
|
| + goto search_success;
|
| + }else{
|
| + goto search_failed;
|
| + }
|
| + }else if( i<nNew && a[iOff+i]>pTerm[nMatch] ){
|
| + goto search_failed;
|
| + }
|
| + }
|
| +
|
| + if( iPgidx>=n ){
|
| + bEndOfPage = 1;
|
| + break;
|
| + }
|
| +
|
| + iPgidx += fts5GetVarint32(&a[iPgidx], nKeep);
|
| + iTermOff += nKeep;
|
| + iOff = iTermOff;
|
| +
|
| + /* Read the nKeep field of the next term. */
|
| + fts5FastGetVarint32(a, iOff, nKeep);
|
| + }
|
| +
|
| + search_failed:
|
| + if( bGe==0 ){
|
| + fts5DataRelease(pIter->pLeaf);
|
| + pIter->pLeaf = 0;
|
| + return;
|
| + }else if( bEndOfPage ){
|
| + do {
|
| + fts5SegIterNextPage(p, pIter);
|
| + if( pIter->pLeaf==0 ) return;
|
| + a = pIter->pLeaf->p;
|
| + if( fts5LeafIsTermless(pIter->pLeaf)==0 ){
|
| + iPgidx = pIter->pLeaf->szLeaf;
|
| + iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff);
|
| + if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){
|
| + p->rc = FTS5_CORRUPT;
|
| + }else{
|
| + nKeep = 0;
|
| + iTermOff = iOff;
|
| + n = pIter->pLeaf->nn;
|
| + iOff += fts5GetVarint32(&a[iOff], nNew);
|
| + break;
|
| + }
|
| + }
|
| + }while( 1 );
|
| + }
|
| +
|
| + search_success:
|
| +
|
| + pIter->iLeafOffset = iOff + nNew;
|
| + pIter->iTermLeafOffset = pIter->iLeafOffset;
|
| + pIter->iTermLeafPgno = pIter->iLeafPgno;
|
| +
|
| + fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm);
|
| + fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]);
|
| +
|
| + if( iPgidx>=n ){
|
| + pIter->iEndofDoclist = pIter->pLeaf->nn+1;
|
| + }else{
|
| + int nExtra;
|
| + iPgidx += fts5GetVarint32(&a[iPgidx], nExtra);
|
| + pIter->iEndofDoclist = iTermOff + nExtra;
|
| + }
|
| + pIter->iPgidxOff = iPgidx;
|
| +
|
| + fts5SegIterLoadRowid(p, pIter);
|
| + fts5SegIterLoadNPos(p, pIter);
|
| +}
|
| +
|
| +/*
|
| +** Initialize the object pIter to point to term pTerm/nTerm within segment
|
| +** pSeg. If there is no such term in the index, the iterator is set to EOF.
|
| +**
|
| +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If
|
| +** an error has already occurred when this function is called, it is a no-op.
|
| +*/
|
| +static void fts5SegIterSeekInit(
|
| + Fts5Index *p, /* FTS5 backend */
|
| + Fts5Buffer *pBuf, /* Buffer to use for loading pages */
|
| + const u8 *pTerm, int nTerm, /* Term to seek to */
|
| + int flags, /* Mask of FTS5INDEX_XXX flags */
|
| + Fts5StructureSegment *pSeg, /* Description of segment */
|
| + Fts5SegIter *pIter /* Object to populate */
|
| +){
|
| + int iPg = 1;
|
| + int bGe = (flags & FTS5INDEX_QUERY_SCAN);
|
| + int bDlidx = 0; /* True if there is a doclist-index */
|
| +
|
| + static int nCall = 0;
|
| + nCall++;
|
| +
|
| + assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 );
|
| + assert( pTerm && nTerm );
|
| + memset(pIter, 0, sizeof(*pIter));
|
| + pIter->pSeg = pSeg;
|
| +
|
| + /* This block sets stack variable iPg to the leaf page number that may
|
| + ** contain term (pTerm/nTerm), if it is present in the segment. */
|
| + if( p->pIdxSelect==0 ){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf(
|
| + "SELECT pgno FROM '%q'.'%q_idx' WHERE "
|
| + "segid=? AND term<=? ORDER BY term DESC LIMIT 1",
|
| + pConfig->zDb, pConfig->zName
|
| + ));
|
| + }
|
| + if( p->rc ) return;
|
| + sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid);
|
| + sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
|
| + if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){
|
| + i64 val = sqlite3_column_int(p->pIdxSelect, 0);
|
| + iPg = (int)(val>>1);
|
| + bDlidx = (val & 0x0001);
|
| + }
|
| + p->rc = sqlite3_reset(p->pIdxSelect);
|
| +
|
| + if( iPg<pSeg->pgnoFirst ){
|
| + iPg = pSeg->pgnoFirst;
|
| + bDlidx = 0;
|
| + }
|
| +
|
| + pIter->iLeafPgno = iPg - 1;
|
| + fts5SegIterNextPage(p, pIter);
|
| +
|
| + if( pIter->pLeaf ){
|
| + fts5LeafSeek(p, bGe, pIter, pTerm, nTerm);
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK && bGe==0 ){
|
| + pIter->flags |= FTS5_SEGITER_ONETERM;
|
| + if( pIter->pLeaf ){
|
| + if( flags & FTS5INDEX_QUERY_DESC ){
|
| + pIter->flags |= FTS5_SEGITER_REVERSE;
|
| + }
|
| + if( bDlidx ){
|
| + fts5SegIterLoadDlidx(p, pIter);
|
| + }
|
| + if( flags & FTS5INDEX_QUERY_DESC ){
|
| + fts5SegIterReverse(p, pIter);
|
| + }
|
| + }
|
| + }
|
| +
|
| + /* Either:
|
| + **
|
| + ** 1) an error has occurred, or
|
| + ** 2) the iterator points to EOF, or
|
| + ** 3) the iterator points to an entry with term (pTerm/nTerm), or
|
| + ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points
|
| + ** to an entry with a term greater than or equal to (pTerm/nTerm).
|
| + */
|
| + assert( p->rc!=SQLITE_OK /* 1 */
|
| + || pIter->pLeaf==0 /* 2 */
|
| + || fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)==0 /* 3 */
|
| + || (bGe && fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0) /* 4 */
|
| + );
|
| +}
|
| +
|
| +/*
|
| +** Initialize the object pIter to point to term pTerm/nTerm within the
|
| +** in-memory hash table. If there is no such term in the hash-table, the
|
| +** iterator is set to EOF.
|
| +**
|
| +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If
|
| +** an error has already occurred when this function is called, it is a no-op.
|
| +*/
|
| +static void fts5SegIterHashInit(
|
| + Fts5Index *p, /* FTS5 backend */
|
| + const u8 *pTerm, int nTerm, /* Term to seek to */
|
| + int flags, /* Mask of FTS5INDEX_XXX flags */
|
| + Fts5SegIter *pIter /* Object to populate */
|
| +){
|
| + const u8 *pList = 0;
|
| + int nList = 0;
|
| + const u8 *z = 0;
|
| + int n = 0;
|
| +
|
| + assert( p->pHash );
|
| + assert( p->rc==SQLITE_OK );
|
| +
|
| + if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){
|
| + p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm);
|
| + sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList);
|
| + n = (z ? (int)strlen((const char*)z) : 0);
|
| + }else{
|
| + pIter->flags |= FTS5_SEGITER_ONETERM;
|
| + sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList);
|
| + z = pTerm;
|
| + n = nTerm;
|
| + }
|
| +
|
| + if( pList ){
|
| + Fts5Data *pLeaf;
|
| + sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z);
|
| + pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data));
|
| + if( pLeaf==0 ) return;
|
| + pLeaf->p = (u8*)pList;
|
| + pLeaf->nn = pLeaf->szLeaf = nList;
|
| + pIter->pLeaf = pLeaf;
|
| + pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid);
|
| + pIter->iEndofDoclist = pLeaf->nn+1;
|
| +
|
| + if( flags & FTS5INDEX_QUERY_DESC ){
|
| + pIter->flags |= FTS5_SEGITER_REVERSE;
|
| + fts5SegIterReverseInitPage(p, pIter);
|
| + }else{
|
| + fts5SegIterLoadNPos(p, pIter);
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Zero the iterator passed as the only argument.
|
| +*/
|
| +static void fts5SegIterClear(Fts5SegIter *pIter){
|
| + fts5BufferFree(&pIter->term);
|
| + fts5DataRelease(pIter->pLeaf);
|
| + fts5DataRelease(pIter->pNextLeaf);
|
| + fts5DlidxIterFree(pIter->pDlidx);
|
| + sqlite3_free(pIter->aRowidOffset);
|
| + memset(pIter, 0, sizeof(Fts5SegIter));
|
| +}
|
| +
|
| +#ifdef SQLITE_DEBUG
|
| +
|
| +/*
|
| +** This function is used as part of the big assert() procedure implemented by
|
| +** fts5AssertMultiIterSetup(). It ensures that the result currently stored
|
| +** in *pRes is the correct result of comparing the current positions of the
|
| +** two iterators.
|
| +*/
|
| +static void fts5AssertComparisonResult(
|
| + Fts5IndexIter *pIter,
|
| + Fts5SegIter *p1,
|
| + Fts5SegIter *p2,
|
| + Fts5CResult *pRes
|
| +){
|
| + int i1 = p1 - pIter->aSeg;
|
| + int i2 = p2 - pIter->aSeg;
|
| +
|
| + if( p1->pLeaf || p2->pLeaf ){
|
| + if( p1->pLeaf==0 ){
|
| + assert( pRes->iFirst==i2 );
|
| + }else if( p2->pLeaf==0 ){
|
| + assert( pRes->iFirst==i1 );
|
| + }else{
|
| + int nMin = MIN(p1->term.n, p2->term.n);
|
| + int res = memcmp(p1->term.p, p2->term.p, nMin);
|
| + if( res==0 ) res = p1->term.n - p2->term.n;
|
| +
|
| + if( res==0 ){
|
| + assert( pRes->bTermEq==1 );
|
| + assert( p1->iRowid!=p2->iRowid );
|
| + res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : 1;
|
| + }else{
|
| + assert( pRes->bTermEq==0 );
|
| + }
|
| +
|
| + if( res<0 ){
|
| + assert( pRes->iFirst==i1 );
|
| + }else{
|
| + assert( pRes->iFirst==i2 );
|
| + }
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** This function is a no-op unless SQLITE_DEBUG is defined when this module
|
| +** is compiled. In that case, this function is essentially an assert()
|
| +** statement used to verify that the contents of the pIter->aFirst[] array
|
| +** are correct.
|
| +*/
|
| +static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){
|
| + if( p->rc==SQLITE_OK ){
|
| + Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
|
| + int i;
|
| +
|
| + assert( (pFirst->pLeaf==0)==pIter->bEof );
|
| +
|
| + /* Check that pIter->iSwitchRowid is set correctly. */
|
| + for(i=0; i<pIter->nSeg; i++){
|
| + Fts5SegIter *p1 = &pIter->aSeg[i];
|
| + assert( p1==pFirst
|
| + || p1->pLeaf==0
|
| + || fts5BufferCompare(&pFirst->term, &p1->term)
|
| + || p1->iRowid==pIter->iSwitchRowid
|
| + || (p1->iRowid<pIter->iSwitchRowid)==pIter->bRev
|
| + );
|
| + }
|
| +
|
| + for(i=0; i<pIter->nSeg; i+=2){
|
| + Fts5SegIter *p1 = &pIter->aSeg[i];
|
| + Fts5SegIter *p2 = &pIter->aSeg[i+1];
|
| + Fts5CResult *pRes = &pIter->aFirst[(pIter->nSeg + i) / 2];
|
| + fts5AssertComparisonResult(pIter, p1, p2, pRes);
|
| + }
|
| +
|
| + for(i=1; i<(pIter->nSeg / 2); i+=2){
|
| + Fts5SegIter *p1 = &pIter->aSeg[ pIter->aFirst[i*2].iFirst ];
|
| + Fts5SegIter *p2 = &pIter->aSeg[ pIter->aFirst[i*2+1].iFirst ];
|
| + Fts5CResult *pRes = &pIter->aFirst[i];
|
| + fts5AssertComparisonResult(pIter, p1, p2, pRes);
|
| + }
|
| + }
|
| +}
|
| +#else
|
| +# define fts5AssertMultiIterSetup(x,y)
|
| +#endif
|
| +
|
| +/*
|
| +** Do the comparison necessary to populate pIter->aFirst[iOut].
|
| +**
|
| +** If the returned value is non-zero, then it is the index of an entry
|
| +** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing
|
| +** to a key that is a duplicate of another, higher priority,
|
| +** segment-iterator in the pSeg->aSeg[] array.
|
| +*/
|
| +static int fts5MultiIterDoCompare(Fts5IndexIter *pIter, int iOut){
|
| + int i1; /* Index of left-hand Fts5SegIter */
|
| + int i2; /* Index of right-hand Fts5SegIter */
|
| + int iRes;
|
| + Fts5SegIter *p1; /* Left-hand Fts5SegIter */
|
| + Fts5SegIter *p2; /* Right-hand Fts5SegIter */
|
| + Fts5CResult *pRes = &pIter->aFirst[iOut];
|
| +
|
| + assert( iOut<pIter->nSeg && iOut>0 );
|
| + assert( pIter->bRev==0 || pIter->bRev==1 );
|
| +
|
| + if( iOut>=(pIter->nSeg/2) ){
|
| + i1 = (iOut - pIter->nSeg/2) * 2;
|
| + i2 = i1 + 1;
|
| + }else{
|
| + i1 = pIter->aFirst[iOut*2].iFirst;
|
| + i2 = pIter->aFirst[iOut*2+1].iFirst;
|
| + }
|
| + p1 = &pIter->aSeg[i1];
|
| + p2 = &pIter->aSeg[i2];
|
| +
|
| + pRes->bTermEq = 0;
|
| + if( p1->pLeaf==0 ){ /* If p1 is at EOF */
|
| + iRes = i2;
|
| + }else if( p2->pLeaf==0 ){ /* If p2 is at EOF */
|
| + iRes = i1;
|
| + }else{
|
| + int res = fts5BufferCompare(&p1->term, &p2->term);
|
| + if( res==0 ){
|
| + assert( i2>i1 );
|
| + assert( i2!=0 );
|
| + pRes->bTermEq = 1;
|
| + if( p1->iRowid==p2->iRowid ){
|
| + p1->bDel = p2->bDel;
|
| + return i2;
|
| + }
|
| + res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1;
|
| + }
|
| + assert( res!=0 );
|
| + if( res<0 ){
|
| + iRes = i1;
|
| + }else{
|
| + iRes = i2;
|
| + }
|
| + }
|
| +
|
| + pRes->iFirst = (u16)iRes;
|
| + return 0;
|
| +}
|
| +
|
| +/*
|
| +** Move the seg-iter so that it points to the first rowid on page iLeafPgno.
|
| +** It is an error if leaf iLeafPgno does not exist or contains no rowids.
|
| +*/
|
| +static void fts5SegIterGotoPage(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5SegIter *pIter, /* Iterator to advance */
|
| + int iLeafPgno
|
| +){
|
| + assert( iLeafPgno>pIter->iLeafPgno );
|
| +
|
| + if( iLeafPgno>pIter->pSeg->pgnoLast ){
|
| + p->rc = FTS5_CORRUPT;
|
| + }else{
|
| + fts5DataRelease(pIter->pNextLeaf);
|
| + pIter->pNextLeaf = 0;
|
| + pIter->iLeafPgno = iLeafPgno-1;
|
| + fts5SegIterNextPage(p, pIter);
|
| + assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno );
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + int iOff;
|
| + u8 *a = pIter->pLeaf->p;
|
| + int n = pIter->pLeaf->szLeaf;
|
| +
|
| + iOff = fts5LeafFirstRowidOff(pIter->pLeaf);
|
| + if( iOff<4 || iOff>=n ){
|
| + p->rc = FTS5_CORRUPT;
|
| + }else{
|
| + iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
|
| + pIter->iLeafOffset = iOff;
|
| + fts5SegIterLoadNPos(p, pIter);
|
| + }
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Advance the iterator passed as the second argument until it is at or
|
| +** past rowid iFrom. Regardless of the value of iFrom, the iterator is
|
| +** always advanced at least once.
|
| +*/
|
| +static void fts5SegIterNextFrom(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5SegIter *pIter, /* Iterator to advance */
|
| + i64 iMatch /* Advance iterator at least this far */
|
| +){
|
| + int bRev = (pIter->flags & FTS5_SEGITER_REVERSE);
|
| + Fts5DlidxIter *pDlidx = pIter->pDlidx;
|
| + int iLeafPgno = pIter->iLeafPgno;
|
| + int bMove = 1;
|
| +
|
| + assert( pIter->flags & FTS5_SEGITER_ONETERM );
|
| + assert( pIter->pDlidx );
|
| + assert( pIter->pLeaf );
|
| +
|
| + if( bRev==0 ){
|
| + while( !fts5DlidxIterEof(p, pDlidx) && iMatch>fts5DlidxIterRowid(pDlidx) ){
|
| + iLeafPgno = fts5DlidxIterPgno(pDlidx);
|
| + fts5DlidxIterNext(p, pDlidx);
|
| + }
|
| + assert_nc( iLeafPgno>=pIter->iLeafPgno || p->rc );
|
| + if( iLeafPgno>pIter->iLeafPgno ){
|
| + fts5SegIterGotoPage(p, pIter, iLeafPgno);
|
| + bMove = 0;
|
| + }
|
| + }else{
|
| + assert( pIter->pNextLeaf==0 );
|
| + assert( iMatch<pIter->iRowid );
|
| + while( !fts5DlidxIterEof(p, pDlidx) && iMatch<fts5DlidxIterRowid(pDlidx) ){
|
| + fts5DlidxIterPrev(p, pDlidx);
|
| + }
|
| + iLeafPgno = fts5DlidxIterPgno(pDlidx);
|
| +
|
| + assert( fts5DlidxIterEof(p, pDlidx) || iLeafPgno<=pIter->iLeafPgno );
|
| +
|
| + if( iLeafPgno<pIter->iLeafPgno ){
|
| + pIter->iLeafPgno = iLeafPgno+1;
|
| + fts5SegIterReverseNewPage(p, pIter);
|
| + bMove = 0;
|
| + }
|
| + }
|
| +
|
| + do{
|
| + if( bMove ) fts5SegIterNext(p, pIter, 0);
|
| + if( pIter->pLeaf==0 ) break;
|
| + if( bRev==0 && pIter->iRowid>=iMatch ) break;
|
| + if( bRev!=0 && pIter->iRowid<=iMatch ) break;
|
| + bMove = 1;
|
| + }while( p->rc==SQLITE_OK );
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Free the iterator object passed as the second argument.
|
| +*/
|
| +static void fts5MultiIterFree(Fts5Index *p, Fts5IndexIter *pIter){
|
| + if( pIter ){
|
| + int i;
|
| + for(i=0; i<pIter->nSeg; i++){
|
| + fts5SegIterClear(&pIter->aSeg[i]);
|
| + }
|
| + fts5StructureRelease(pIter->pStruct);
|
| + fts5BufferFree(&pIter->poslist);
|
| + sqlite3_free(pIter);
|
| + }
|
| +}
|
| +
|
| +static void fts5MultiIterAdvanced(
|
| + Fts5Index *p, /* FTS5 backend to iterate within */
|
| + Fts5IndexIter *pIter, /* Iterator to update aFirst[] array for */
|
| + int iChanged, /* Index of sub-iterator just advanced */
|
| + int iMinset /* Minimum entry in aFirst[] to set */
|
| +){
|
| + int i;
|
| + for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){
|
| + int iEq;
|
| + if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){
|
| + fts5SegIterNext(p, &pIter->aSeg[iEq], 0);
|
| + i = pIter->nSeg + iEq;
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Sub-iterator iChanged of iterator pIter has just been advanced. It still
|
| +** points to the same term though - just a different rowid. This function
|
| +** attempts to update the contents of the pIter->aFirst[] accordingly.
|
| +** If it does so successfully, 0 is returned. Otherwise 1.
|
| +**
|
| +** If non-zero is returned, the caller should call fts5MultiIterAdvanced()
|
| +** on the iterator instead. That function does the same as this one, except
|
| +** that it deals with more complicated cases as well.
|
| +*/
|
| +static int fts5MultiIterAdvanceRowid(
|
| + Fts5Index *p, /* FTS5 backend to iterate within */
|
| + Fts5IndexIter *pIter, /* Iterator to update aFirst[] array for */
|
| + int iChanged /* Index of sub-iterator just advanced */
|
| +){
|
| + Fts5SegIter *pNew = &pIter->aSeg[iChanged];
|
| +
|
| + if( pNew->iRowid==pIter->iSwitchRowid
|
| + || (pNew->iRowid<pIter->iSwitchRowid)==pIter->bRev
|
| + ){
|
| + int i;
|
| + Fts5SegIter *pOther = &pIter->aSeg[iChanged ^ 0x0001];
|
| + pIter->iSwitchRowid = pIter->bRev ? SMALLEST_INT64 : LARGEST_INT64;
|
| + for(i=(pIter->nSeg+iChanged)/2; 1; i=i/2){
|
| + Fts5CResult *pRes = &pIter->aFirst[i];
|
| +
|
| + assert( pNew->pLeaf );
|
| + assert( pRes->bTermEq==0 || pOther->pLeaf );
|
| +
|
| + if( pRes->bTermEq ){
|
| + if( pNew->iRowid==pOther->iRowid ){
|
| + return 1;
|
| + }else if( (pOther->iRowid>pNew->iRowid)==pIter->bRev ){
|
| + pIter->iSwitchRowid = pOther->iRowid;
|
| + pNew = pOther;
|
| + }else if( (pOther->iRowid>pIter->iSwitchRowid)==pIter->bRev ){
|
| + pIter->iSwitchRowid = pOther->iRowid;
|
| + }
|
| + }
|
| + pRes->iFirst = (u16)(pNew - pIter->aSeg);
|
| + if( i==1 ) break;
|
| +
|
| + pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ];
|
| + }
|
| + }
|
| +
|
| + return 0;
|
| +}
|
| +
|
| +/*
|
| +** Set the pIter->bEof variable based on the state of the sub-iterators.
|
| +*/
|
| +static void fts5MultiIterSetEof(Fts5IndexIter *pIter){
|
| + Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
|
| + pIter->bEof = pSeg->pLeaf==0;
|
| + pIter->iSwitchRowid = pSeg->iRowid;
|
| +}
|
| +
|
| +/*
|
| +** Move the iterator to the next entry.
|
| +**
|
| +** If an error occurs, an error code is left in Fts5Index.rc. It is not
|
| +** considered an error if the iterator reaches EOF, or if it is already at
|
| +** EOF when this function is called.
|
| +*/
|
| +static void fts5MultiIterNext(
|
| + Fts5Index *p,
|
| + Fts5IndexIter *pIter,
|
| + int bFrom, /* True if argument iFrom is valid */
|
| + i64 iFrom /* Advance at least as far as this */
|
| +){
|
| + if( p->rc==SQLITE_OK ){
|
| + int bUseFrom = bFrom;
|
| + do {
|
| + int iFirst = pIter->aFirst[1].iFirst;
|
| + int bNewTerm = 0;
|
| + Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
|
| + assert( p->rc==SQLITE_OK );
|
| + if( bUseFrom && pSeg->pDlidx ){
|
| + fts5SegIterNextFrom(p, pSeg, iFrom);
|
| + }else{
|
| + fts5SegIterNext(p, pSeg, &bNewTerm);
|
| + }
|
| +
|
| + if( pSeg->pLeaf==0 || bNewTerm
|
| + || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
|
| + ){
|
| + fts5MultiIterAdvanced(p, pIter, iFirst, 1);
|
| + fts5MultiIterSetEof(pIter);
|
| + }
|
| + fts5AssertMultiIterSetup(p, pIter);
|
| +
|
| + bUseFrom = 0;
|
| + }while( pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter) );
|
| + }
|
| +}
|
| +
|
| +static void fts5MultiIterNext2(
|
| + Fts5Index *p,
|
| + Fts5IndexIter *pIter,
|
| + int *pbNewTerm /* OUT: True if *might* be new term */
|
| +){
|
| + assert( pIter->bSkipEmpty );
|
| + if( p->rc==SQLITE_OK ){
|
| + do {
|
| + int iFirst = pIter->aFirst[1].iFirst;
|
| + Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
|
| + int bNewTerm = 0;
|
| +
|
| + fts5SegIterNext(p, pSeg, &bNewTerm);
|
| + if( pSeg->pLeaf==0 || bNewTerm
|
| + || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
|
| + ){
|
| + fts5MultiIterAdvanced(p, pIter, iFirst, 1);
|
| + fts5MultiIterSetEof(pIter);
|
| + *pbNewTerm = 1;
|
| + }else{
|
| + *pbNewTerm = 0;
|
| + }
|
| + fts5AssertMultiIterSetup(p, pIter);
|
| +
|
| + }while( fts5MultiIterIsEmpty(p, pIter) );
|
| + }
|
| +}
|
| +
|
| +
|
| +static Fts5IndexIter *fts5MultiIterAlloc(
|
| + Fts5Index *p, /* FTS5 backend to iterate within */
|
| + int nSeg
|
| +){
|
| + Fts5IndexIter *pNew;
|
| + int nSlot; /* Power of two >= nSeg */
|
| +
|
| + for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
|
| + pNew = fts5IdxMalloc(p,
|
| + sizeof(Fts5IndexIter) + /* pNew */
|
| + sizeof(Fts5SegIter) * (nSlot-1) + /* pNew->aSeg[] */
|
| + sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */
|
| + );
|
| + if( pNew ){
|
| + pNew->nSeg = nSlot;
|
| + pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot];
|
| + pNew->pIndex = p;
|
| + }
|
| + return pNew;
|
| +}
|
| +
|
| +/*
|
| +** Allocate a new Fts5IndexIter object.
|
| +**
|
| +** The new object will be used to iterate through data in structure pStruct.
|
| +** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel
|
| +** is zero or greater, data from the first nSegment segments on level iLevel
|
| +** is merged.
|
| +**
|
| +** The iterator initially points to the first term/rowid entry in the
|
| +** iterated data.
|
| +*/
|
| +static void fts5MultiIterNew(
|
| + Fts5Index *p, /* FTS5 backend to iterate within */
|
| + Fts5Structure *pStruct, /* Structure of specific index */
|
| + int bSkipEmpty, /* True to ignore delete-keys */
|
| + int flags, /* FTS5INDEX_QUERY_XXX flags */
|
| + const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */
|
| + int iLevel, /* Level to iterate (-1 for all) */
|
| + int nSegment, /* Number of segments to merge (iLevel>=0) */
|
| + Fts5IndexIter **ppOut /* New object */
|
| +){
|
| + int nSeg = 0; /* Number of segment-iters in use */
|
| + int iIter = 0; /* */
|
| + int iSeg; /* Used to iterate through segments */
|
| + Fts5Buffer buf = {0,0,0}; /* Buffer used by fts5SegIterSeekInit() */
|
| + Fts5StructureLevel *pLvl;
|
| + Fts5IndexIter *pNew;
|
| +
|
| + assert( (pTerm==0 && nTerm==0) || iLevel<0 );
|
| +
|
| + /* Allocate space for the new multi-seg-iterator. */
|
| + if( p->rc==SQLITE_OK ){
|
| + if( iLevel<0 ){
|
| + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
|
| + nSeg = pStruct->nSegment;
|
| + nSeg += (p->pHash ? 1 : 0);
|
| + }else{
|
| + nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
|
| + }
|
| + }
|
| + *ppOut = pNew = fts5MultiIterAlloc(p, nSeg);
|
| + if( pNew==0 ) return;
|
| + pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC));
|
| + pNew->bSkipEmpty = (u8)bSkipEmpty;
|
| + pNew->pStruct = pStruct;
|
| + fts5StructureRef(pStruct);
|
| +
|
| + /* Initialize each of the component segment iterators. */
|
| + if( iLevel<0 ){
|
| + Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
|
| + if( p->pHash ){
|
| + /* Add a segment iterator for the current contents of the hash table. */
|
| + Fts5SegIter *pIter = &pNew->aSeg[iIter++];
|
| + fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
|
| + }
|
| + for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){
|
| + for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){
|
| + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
|
| + Fts5SegIter *pIter = &pNew->aSeg[iIter++];
|
| + if( pTerm==0 ){
|
| + fts5SegIterInit(p, pSeg, pIter);
|
| + }else{
|
| + fts5SegIterSeekInit(p, &buf, pTerm, nTerm, flags, pSeg, pIter);
|
| + }
|
| + }
|
| + }
|
| + }else{
|
| + pLvl = &pStruct->aLevel[iLevel];
|
| + for(iSeg=nSeg-1; iSeg>=0; iSeg--){
|
| + fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]);
|
| + }
|
| + }
|
| + assert( iIter==nSeg );
|
| +
|
| + /* If the above was successful, each component iterators now points
|
| + ** to the first entry in its segment. In this case initialize the
|
| + ** aFirst[] array. Or, if an error has occurred, free the iterator
|
| + ** object and set the output variable to NULL. */
|
| + if( p->rc==SQLITE_OK ){
|
| + for(iIter=pNew->nSeg-1; iIter>0; iIter--){
|
| + int iEq;
|
| + if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){
|
| + fts5SegIterNext(p, &pNew->aSeg[iEq], 0);
|
| + fts5MultiIterAdvanced(p, pNew, iEq, iIter);
|
| + }
|
| + }
|
| + fts5MultiIterSetEof(pNew);
|
| + fts5AssertMultiIterSetup(p, pNew);
|
| +
|
| + if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
|
| + fts5MultiIterNext(p, pNew, 0, 0);
|
| + }
|
| + }else{
|
| + fts5MultiIterFree(p, pNew);
|
| + *ppOut = 0;
|
| + }
|
| + fts5BufferFree(&buf);
|
| +}
|
| +
|
| +/*
|
| +** Create an Fts5IndexIter that iterates through the doclist provided
|
| +** as the second argument.
|
| +*/
|
| +static void fts5MultiIterNew2(
|
| + Fts5Index *p, /* FTS5 backend to iterate within */
|
| + Fts5Data *pData, /* Doclist to iterate through */
|
| + int bDesc, /* True for descending rowid order */
|
| + Fts5IndexIter **ppOut /* New object */
|
| +){
|
| + Fts5IndexIter *pNew;
|
| + pNew = fts5MultiIterAlloc(p, 2);
|
| + if( pNew ){
|
| + Fts5SegIter *pIter = &pNew->aSeg[1];
|
| +
|
| + pNew->bFiltered = 1;
|
| + pIter->flags = FTS5_SEGITER_ONETERM;
|
| + if( pData->szLeaf>0 ){
|
| + pIter->pLeaf = pData;
|
| + pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid);
|
| + pIter->iEndofDoclist = pData->nn;
|
| + pNew->aFirst[1].iFirst = 1;
|
| + if( bDesc ){
|
| + pNew->bRev = 1;
|
| + pIter->flags |= FTS5_SEGITER_REVERSE;
|
| + fts5SegIterReverseInitPage(p, pIter);
|
| + }else{
|
| + fts5SegIterLoadNPos(p, pIter);
|
| + }
|
| + pData = 0;
|
| + }else{
|
| + pNew->bEof = 1;
|
| + }
|
| +
|
| + *ppOut = pNew;
|
| + }
|
| +
|
| + fts5DataRelease(pData);
|
| +}
|
| +
|
| +/*
|
| +** Return true if the iterator is at EOF or if an error has occurred.
|
| +** False otherwise.
|
| +*/
|
| +static int fts5MultiIterEof(Fts5Index *p, Fts5IndexIter *pIter){
|
| + assert( p->rc
|
| + || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof
|
| + );
|
| + return (p->rc || pIter->bEof);
|
| +}
|
| +
|
| +/*
|
| +** Return the rowid of the entry that the iterator currently points
|
| +** to. If the iterator points to EOF when this function is called the
|
| +** results are undefined.
|
| +*/
|
| +static i64 fts5MultiIterRowid(Fts5IndexIter *pIter){
|
| + assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf );
|
| + return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid;
|
| +}
|
| +
|
| +/*
|
| +** Move the iterator to the next entry at or following iMatch.
|
| +*/
|
| +static void fts5MultiIterNextFrom(
|
| + Fts5Index *p,
|
| + Fts5IndexIter *pIter,
|
| + i64 iMatch
|
| +){
|
| + while( 1 ){
|
| + i64 iRowid;
|
| + fts5MultiIterNext(p, pIter, 1, iMatch);
|
| + if( fts5MultiIterEof(p, pIter) ) break;
|
| + iRowid = fts5MultiIterRowid(pIter);
|
| + if( pIter->bRev==0 && iRowid>=iMatch ) break;
|
| + if( pIter->bRev!=0 && iRowid<=iMatch ) break;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Return a pointer to a buffer containing the term associated with the
|
| +** entry that the iterator currently points to.
|
| +*/
|
| +static const u8 *fts5MultiIterTerm(Fts5IndexIter *pIter, int *pn){
|
| + Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
|
| + *pn = p->term.n;
|
| + return p->term.p;
|
| +}
|
| +
|
| +static void fts5ChunkIterate(
|
| + Fts5Index *p, /* Index object */
|
| + Fts5SegIter *pSeg, /* Poslist of this iterator */
|
| + void *pCtx, /* Context pointer for xChunk callback */
|
| + void (*xChunk)(Fts5Index*, void*, const u8*, int)
|
| +){
|
| + int nRem = pSeg->nPos; /* Number of bytes still to come */
|
| + Fts5Data *pData = 0;
|
| + u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset];
|
| + int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset);
|
| + int pgno = pSeg->iLeafPgno;
|
| + int pgnoSave = 0;
|
| +
|
| + if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){
|
| + pgnoSave = pgno+1;
|
| + }
|
| +
|
| + while( 1 ){
|
| + xChunk(p, pCtx, pChunk, nChunk);
|
| + nRem -= nChunk;
|
| + fts5DataRelease(pData);
|
| + if( nRem<=0 ){
|
| + break;
|
| + }else{
|
| + pgno++;
|
| + pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno));
|
| + if( pData==0 ) break;
|
| + pChunk = &pData->p[4];
|
| + nChunk = MIN(nRem, pData->szLeaf - 4);
|
| + if( pgno==pgnoSave ){
|
| + assert( pSeg->pNextLeaf==0 );
|
| + pSeg->pNextLeaf = pData;
|
| + pData = 0;
|
| + }
|
| + }
|
| + }
|
| +}
|
| +
|
| +
|
| +
|
| +/*
|
| +** Allocate a new segment-id for the structure pStruct. The new segment
|
| +** id must be between 1 and 65335 inclusive, and must not be used by
|
| +** any currently existing segment. If a free segment id cannot be found,
|
| +** SQLITE_FULL is returned.
|
| +**
|
| +** If an error has already occurred, this function is a no-op. 0 is
|
| +** returned in this case.
|
| +*/
|
| +static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
|
| + int iSegid = 0;
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){
|
| + p->rc = SQLITE_FULL;
|
| + }else{
|
| + while( iSegid==0 ){
|
| + int iLvl, iSeg;
|
| + sqlite3_randomness(sizeof(u32), (void*)&iSegid);
|
| + iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1);
|
| + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
|
| + for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
|
| + if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){
|
| + iSegid = 0;
|
| + }
|
| + }
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + return iSegid;
|
| +}
|
| +
|
| +/*
|
| +** Discard all data currently cached in the hash-tables.
|
| +*/
|
| +static void fts5IndexDiscardData(Fts5Index *p){
|
| + assert( p->pHash || p->nPendingData==0 );
|
| + if( p->pHash ){
|
| + sqlite3Fts5HashClear(p->pHash);
|
| + p->nPendingData = 0;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Return the size of the prefix, in bytes, that buffer (nNew/pNew) shares
|
| +** with buffer (nOld/pOld).
|
| +*/
|
| +static int fts5PrefixCompress(
|
| + int nOld, const u8 *pOld,
|
| + int nNew, const u8 *pNew
|
| +){
|
| + int i;
|
| + assert( fts5BlobCompare(pOld, nOld, pNew, nNew)<0 );
|
| + for(i=0; i<nOld; i++){
|
| + if( pOld[i]!=pNew[i] ) break;
|
| + }
|
| + return i;
|
| +}
|
| +
|
| +static void fts5WriteDlidxClear(
|
| + Fts5Index *p,
|
| + Fts5SegWriter *pWriter,
|
| + int bFlush /* If true, write dlidx to disk */
|
| +){
|
| + int i;
|
| + assert( bFlush==0 || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n>0) );
|
| + for(i=0; i<pWriter->nDlidx; i++){
|
| + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i];
|
| + if( pDlidx->buf.n==0 ) break;
|
| + if( bFlush ){
|
| + assert( pDlidx->pgno!=0 );
|
| + fts5DataWrite(p,
|
| + FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno),
|
| + pDlidx->buf.p, pDlidx->buf.n
|
| + );
|
| + }
|
| + sqlite3Fts5BufferZero(&pDlidx->buf);
|
| + pDlidx->bPrevValid = 0;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Grow the pWriter->aDlidx[] array to at least nLvl elements in size.
|
| +** Any new array elements are zeroed before returning.
|
| +*/
|
| +static int fts5WriteDlidxGrow(
|
| + Fts5Index *p,
|
| + Fts5SegWriter *pWriter,
|
| + int nLvl
|
| +){
|
| + if( p->rc==SQLITE_OK && nLvl>=pWriter->nDlidx ){
|
| + Fts5DlidxWriter *aDlidx = (Fts5DlidxWriter*)sqlite3_realloc(
|
| + pWriter->aDlidx, sizeof(Fts5DlidxWriter) * nLvl
|
| + );
|
| + if( aDlidx==0 ){
|
| + p->rc = SQLITE_NOMEM;
|
| + }else{
|
| + int nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx);
|
| + memset(&aDlidx[pWriter->nDlidx], 0, nByte);
|
| + pWriter->aDlidx = aDlidx;
|
| + pWriter->nDlidx = nLvl;
|
| + }
|
| + }
|
| + return p->rc;
|
| +}
|
| +
|
| +/*
|
| +** If the current doclist-index accumulating in pWriter->aDlidx[] is large
|
| +** enough, flush it to disk and return 1. Otherwise discard it and return
|
| +** zero.
|
| +*/
|
| +static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){
|
| + int bFlag = 0;
|
| +
|
| + /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written
|
| + ** to the database, also write the doclist-index to disk. */
|
| + if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
|
| + bFlag = 1;
|
| + }
|
| + fts5WriteDlidxClear(p, pWriter, bFlag);
|
| + pWriter->nEmpty = 0;
|
| + return bFlag;
|
| +}
|
| +
|
| +/*
|
| +** This function is called whenever processing of the doclist for the
|
| +** last term on leaf page (pWriter->iBtPage) is completed.
|
| +**
|
| +** The doclist-index for that term is currently stored in-memory within the
|
| +** Fts5SegWriter.aDlidx[] array. If it is large enough, this function
|
| +** writes it out to disk. Or, if it is too small to bother with, discards
|
| +** it.
|
| +**
|
| +** Fts5SegWriter.btterm currently contains the first term on page iBtPage.
|
| +*/
|
| +static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){
|
| + int bFlag;
|
| +
|
| + assert( pWriter->iBtPage || pWriter->nEmpty==0 );
|
| + if( pWriter->iBtPage==0 ) return;
|
| + bFlag = fts5WriteFlushDlidx(p, pWriter);
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:"");
|
| + /* The following was already done in fts5WriteInit(): */
|
| + /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */
|
| + sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC);
|
| + sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1));
|
| + sqlite3_step(p->pIdxWriter);
|
| + p->rc = sqlite3_reset(p->pIdxWriter);
|
| + }
|
| + pWriter->iBtPage = 0;
|
| +}
|
| +
|
| +/*
|
| +** This is called once for each leaf page except the first that contains
|
| +** at least one term. Argument (nTerm/pTerm) is the split-key - a term that
|
| +** is larger than all terms written to earlier leaves, and equal to or
|
| +** smaller than the first term on the new leaf.
|
| +**
|
| +** If an error occurs, an error code is left in Fts5Index.rc. If an error
|
| +** has already occurred when this function is called, it is a no-op.
|
| +*/
|
| +static void fts5WriteBtreeTerm(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5SegWriter *pWriter, /* Writer object */
|
| + int nTerm, const u8 *pTerm /* First term on new page */
|
| +){
|
| + fts5WriteFlushBtree(p, pWriter);
|
| + fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm);
|
| + pWriter->iBtPage = pWriter->writer.pgno;
|
| +}
|
| +
|
| +/*
|
| +** This function is called when flushing a leaf page that contains no
|
| +** terms at all to disk.
|
| +*/
|
| +static void fts5WriteBtreeNoTerm(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5SegWriter *pWriter /* Writer object */
|
| +){
|
| + /* If there were no rowids on the leaf page either and the doclist-index
|
| + ** has already been started, append an 0x00 byte to it. */
|
| + if( pWriter->bFirstRowidInPage && pWriter->aDlidx[0].buf.n>0 ){
|
| + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[0];
|
| + assert( pDlidx->bPrevValid );
|
| + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, 0);
|
| + }
|
| +
|
| + /* Increment the "number of sequential leaves without a term" counter. */
|
| + pWriter->nEmpty++;
|
| +}
|
| +
|
| +static i64 fts5DlidxExtractFirstRowid(Fts5Buffer *pBuf){
|
| + i64 iRowid;
|
| + int iOff;
|
| +
|
| + iOff = 1 + fts5GetVarint(&pBuf->p[1], (u64*)&iRowid);
|
| + fts5GetVarint(&pBuf->p[iOff], (u64*)&iRowid);
|
| + return iRowid;
|
| +}
|
| +
|
| +/*
|
| +** Rowid iRowid has just been appended to the current leaf page. It is the
|
| +** first on the page. This function appends an appropriate entry to the current
|
| +** doclist-index.
|
| +*/
|
| +static void fts5WriteDlidxAppend(
|
| + Fts5Index *p,
|
| + Fts5SegWriter *pWriter,
|
| + i64 iRowid
|
| +){
|
| + int i;
|
| + int bDone = 0;
|
| +
|
| + for(i=0; p->rc==SQLITE_OK && bDone==0; i++){
|
| + i64 iVal;
|
| + Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i];
|
| +
|
| + if( pDlidx->buf.n>=p->pConfig->pgsz ){
|
| + /* The current doclist-index page is full. Write it to disk and push
|
| + ** a copy of iRowid (which will become the first rowid on the next
|
| + ** doclist-index leaf page) up into the next level of the b-tree
|
| + ** hierarchy. If the node being flushed is currently the root node,
|
| + ** also push its first rowid upwards. */
|
| + pDlidx->buf.p[0] = 0x01; /* Not the root node */
|
| + fts5DataWrite(p,
|
| + FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno),
|
| + pDlidx->buf.p, pDlidx->buf.n
|
| + );
|
| + fts5WriteDlidxGrow(p, pWriter, i+2);
|
| + pDlidx = &pWriter->aDlidx[i];
|
| + if( p->rc==SQLITE_OK && pDlidx[1].buf.n==0 ){
|
| + i64 iFirst = fts5DlidxExtractFirstRowid(&pDlidx->buf);
|
| +
|
| + /* This was the root node. Push its first rowid up to the new root. */
|
| + pDlidx[1].pgno = pDlidx->pgno;
|
| + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, 0);
|
| + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, pDlidx->pgno);
|
| + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, iFirst);
|
| + pDlidx[1].bPrevValid = 1;
|
| + pDlidx[1].iPrev = iFirst;
|
| + }
|
| +
|
| + sqlite3Fts5BufferZero(&pDlidx->buf);
|
| + pDlidx->bPrevValid = 0;
|
| + pDlidx->pgno++;
|
| + }else{
|
| + bDone = 1;
|
| + }
|
| +
|
| + if( pDlidx->bPrevValid ){
|
| + iVal = iRowid - pDlidx->iPrev;
|
| + }else{
|
| + i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno);
|
| + assert( pDlidx->buf.n==0 );
|
| + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone);
|
| + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno);
|
| + iVal = iRowid;
|
| + }
|
| +
|
| + sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iVal);
|
| + pDlidx->bPrevValid = 1;
|
| + pDlidx->iPrev = iRowid;
|
| + }
|
| +}
|
| +
|
| +static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
|
| + static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
|
| + Fts5PageWriter *pPage = &pWriter->writer;
|
| + i64 iRowid;
|
| +
|
| + assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) );
|
| +
|
| + /* Set the szLeaf header field. */
|
| + assert( 0==fts5GetU16(&pPage->buf.p[2]) );
|
| + fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n);
|
| +
|
| + if( pWriter->bFirstTermInPage ){
|
| + /* No term was written to this page. */
|
| + assert( pPage->pgidx.n==0 );
|
| + fts5WriteBtreeNoTerm(p, pWriter);
|
| + }else{
|
| + /* Append the pgidx to the page buffer. Set the szLeaf header field. */
|
| + fts5BufferAppendBlob(&p->rc, &pPage->buf, pPage->pgidx.n, pPage->pgidx.p);
|
| + }
|
| +
|
| + /* Write the page out to disk */
|
| + iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, pPage->pgno);
|
| + fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n);
|
| +
|
| + /* Initialize the next page. */
|
| + fts5BufferZero(&pPage->buf);
|
| + fts5BufferZero(&pPage->pgidx);
|
| + fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero);
|
| + pPage->iPrevPgidx = 0;
|
| + pPage->pgno++;
|
| +
|
| + /* Increase the leaves written counter */
|
| + pWriter->nLeafWritten++;
|
| +
|
| + /* The new leaf holds no terms or rowids */
|
| + pWriter->bFirstTermInPage = 1;
|
| + pWriter->bFirstRowidInPage = 1;
|
| +}
|
| +
|
| +/*
|
| +** Append term pTerm/nTerm to the segment being written by the writer passed
|
| +** as the second argument.
|
| +**
|
| +** If an error occurs, set the Fts5Index.rc error code. If an error has
|
| +** already occurred, this function is a no-op.
|
| +*/
|
| +static void fts5WriteAppendTerm(
|
| + Fts5Index *p,
|
| + Fts5SegWriter *pWriter,
|
| + int nTerm, const u8 *pTerm
|
| +){
|
| + int nPrefix; /* Bytes of prefix compression for term */
|
| + Fts5PageWriter *pPage = &pWriter->writer;
|
| + Fts5Buffer *pPgidx = &pWriter->writer.pgidx;
|
| +
|
| + assert( p->rc==SQLITE_OK );
|
| + assert( pPage->buf.n>=4 );
|
| + assert( pPage->buf.n>4 || pWriter->bFirstTermInPage );
|
| +
|
| + /* If the current leaf page is full, flush it to disk. */
|
| + if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){
|
| + if( pPage->buf.n>4 ){
|
| + fts5WriteFlushLeaf(p, pWriter);
|
| + }
|
| + fts5BufferGrow(&p->rc, &pPage->buf, nTerm+FTS5_DATA_PADDING);
|
| + }
|
| +
|
| + /* TODO1: Updating pgidx here. */
|
| + pPgidx->n += sqlite3Fts5PutVarint(
|
| + &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx
|
| + );
|
| + pPage->iPrevPgidx = pPage->buf.n;
|
| +#if 0
|
| + fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n);
|
| + pPgidx->n += 2;
|
| +#endif
|
| +
|
| + if( pWriter->bFirstTermInPage ){
|
| + nPrefix = 0;
|
| + if( pPage->pgno!=1 ){
|
| + /* This is the first term on a leaf that is not the leftmost leaf in
|
| + ** the segment b-tree. In this case it is necessary to add a term to
|
| + ** the b-tree hierarchy that is (a) larger than the largest term
|
| + ** already written to the segment and (b) smaller than or equal to
|
| + ** this term. In other words, a prefix of (pTerm/nTerm) that is one
|
| + ** byte longer than the longest prefix (pTerm/nTerm) shares with the
|
| + ** previous term.
|
| + **
|
| + ** Usually, the previous term is available in pPage->term. The exception
|
| + ** is if this is the first term written in an incremental-merge step.
|
| + ** In this case the previous term is not available, so just write a
|
| + ** copy of (pTerm/nTerm) into the parent node. This is slightly
|
| + ** inefficient, but still correct. */
|
| + int n = nTerm;
|
| + if( pPage->term.n ){
|
| + n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
|
| + }
|
| + fts5WriteBtreeTerm(p, pWriter, n, pTerm);
|
| + pPage = &pWriter->writer;
|
| + }
|
| + }else{
|
| + nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
|
| + fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix);
|
| + }
|
| +
|
| + /* Append the number of bytes of new data, then the term data itself
|
| + ** to the page. */
|
| + fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix);
|
| + fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix]);
|
| +
|
| + /* Update the Fts5PageWriter.term field. */
|
| + fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm);
|
| + pWriter->bFirstTermInPage = 0;
|
| +
|
| + pWriter->bFirstRowidInPage = 0;
|
| + pWriter->bFirstRowidInDoclist = 1;
|
| +
|
| + assert( p->rc || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n==0) );
|
| + pWriter->aDlidx[0].pgno = pPage->pgno;
|
| +}
|
| +
|
| +/*
|
| +** Append a rowid and position-list size field to the writers output.
|
| +*/
|
| +static void fts5WriteAppendRowid(
|
| + Fts5Index *p,
|
| + Fts5SegWriter *pWriter,
|
| + i64 iRowid,
|
| + int nPos
|
| +){
|
| + if( p->rc==SQLITE_OK ){
|
| + Fts5PageWriter *pPage = &pWriter->writer;
|
| +
|
| + if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){
|
| + fts5WriteFlushLeaf(p, pWriter);
|
| + }
|
| +
|
| + /* If this is to be the first rowid written to the page, set the
|
| + ** rowid-pointer in the page-header. Also append a value to the dlidx
|
| + ** buffer, in case a doclist-index is required. */
|
| + if( pWriter->bFirstRowidInPage ){
|
| + fts5PutU16(pPage->buf.p, (u16)pPage->buf.n);
|
| + fts5WriteDlidxAppend(p, pWriter, iRowid);
|
| + }
|
| +
|
| + /* Write the rowid. */
|
| + if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){
|
| + fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid);
|
| + }else{
|
| + assert( p->rc || iRowid>pWriter->iPrevRowid );
|
| + fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid);
|
| + }
|
| + pWriter->iPrevRowid = iRowid;
|
| + pWriter->bFirstRowidInDoclist = 0;
|
| + pWriter->bFirstRowidInPage = 0;
|
| +
|
| + fts5BufferAppendVarint(&p->rc, &pPage->buf, nPos);
|
| + }
|
| +}
|
| +
|
| +static void fts5WriteAppendPoslistData(
|
| + Fts5Index *p,
|
| + Fts5SegWriter *pWriter,
|
| + const u8 *aData,
|
| + int nData
|
| +){
|
| + Fts5PageWriter *pPage = &pWriter->writer;
|
| + const u8 *a = aData;
|
| + int n = nData;
|
| +
|
| + assert( p->pConfig->pgsz>0 );
|
| + while( p->rc==SQLITE_OK
|
| + && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz
|
| + ){
|
| + int nReq = p->pConfig->pgsz - pPage->buf.n - pPage->pgidx.n;
|
| + int nCopy = 0;
|
| + while( nCopy<nReq ){
|
| + i64 dummy;
|
| + nCopy += fts5GetVarint(&a[nCopy], (u64*)&dummy);
|
| + }
|
| + fts5BufferAppendBlob(&p->rc, &pPage->buf, nCopy, a);
|
| + a += nCopy;
|
| + n -= nCopy;
|
| + fts5WriteFlushLeaf(p, pWriter);
|
| + }
|
| + if( n>0 ){
|
| + fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Flush any data cached by the writer object to the database. Free any
|
| +** allocations associated with the writer.
|
| +*/
|
| +static void fts5WriteFinish(
|
| + Fts5Index *p,
|
| + Fts5SegWriter *pWriter, /* Writer object */
|
| + int *pnLeaf /* OUT: Number of leaf pages in b-tree */
|
| +){
|
| + int i;
|
| + Fts5PageWriter *pLeaf = &pWriter->writer;
|
| + if( p->rc==SQLITE_OK ){
|
| + assert( pLeaf->pgno>=1 );
|
| + if( pLeaf->buf.n>4 ){
|
| + fts5WriteFlushLeaf(p, pWriter);
|
| + }
|
| + *pnLeaf = pLeaf->pgno-1;
|
| + fts5WriteFlushBtree(p, pWriter);
|
| + }
|
| + fts5BufferFree(&pLeaf->term);
|
| + fts5BufferFree(&pLeaf->buf);
|
| + fts5BufferFree(&pLeaf->pgidx);
|
| + fts5BufferFree(&pWriter->btterm);
|
| +
|
| + for(i=0; i<pWriter->nDlidx; i++){
|
| + sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf);
|
| + }
|
| + sqlite3_free(pWriter->aDlidx);
|
| +}
|
| +
|
| +static void fts5WriteInit(
|
| + Fts5Index *p,
|
| + Fts5SegWriter *pWriter,
|
| + int iSegid
|
| +){
|
| + const int nBuffer = p->pConfig->pgsz + FTS5_DATA_PADDING;
|
| +
|
| + memset(pWriter, 0, sizeof(Fts5SegWriter));
|
| + pWriter->iSegid = iSegid;
|
| +
|
| + fts5WriteDlidxGrow(p, pWriter, 1);
|
| + pWriter->writer.pgno = 1;
|
| + pWriter->bFirstTermInPage = 1;
|
| + pWriter->iBtPage = 1;
|
| +
|
| + assert( pWriter->writer.buf.n==0 );
|
| + assert( pWriter->writer.pgidx.n==0 );
|
| +
|
| + /* Grow the two buffers to pgsz + padding bytes in size. */
|
| + sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.pgidx, nBuffer);
|
| + sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.buf, nBuffer);
|
| +
|
| + if( p->pIdxWriter==0 ){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf(
|
| + "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)",
|
| + pConfig->zDb, pConfig->zName
|
| + ));
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + /* Initialize the 4-byte leaf-page header to 0x00. */
|
| + memset(pWriter->writer.buf.p, 0, 4);
|
| + pWriter->writer.buf.n = 4;
|
| +
|
| + /* Bind the current output segment id to the index-writer. This is an
|
| + ** optimization over binding the same value over and over as rows are
|
| + ** inserted into %_idx by the current writer. */
|
| + sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Iterator pIter was used to iterate through the input segments of on an
|
| +** incremental merge operation. This function is called if the incremental
|
| +** merge step has finished but the input has not been completely exhausted.
|
| +*/
|
| +static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){
|
| + int i;
|
| + Fts5Buffer buf;
|
| + memset(&buf, 0, sizeof(Fts5Buffer));
|
| + for(i=0; i<pIter->nSeg; i++){
|
| + Fts5SegIter *pSeg = &pIter->aSeg[i];
|
| + if( pSeg->pSeg==0 ){
|
| + /* no-op */
|
| + }else if( pSeg->pLeaf==0 ){
|
| + /* All keys from this input segment have been transfered to the output.
|
| + ** Set both the first and last page-numbers to 0 to indicate that the
|
| + ** segment is now empty. */
|
| + pSeg->pSeg->pgnoLast = 0;
|
| + pSeg->pSeg->pgnoFirst = 0;
|
| + }else{
|
| + int iOff = pSeg->iTermLeafOffset; /* Offset on new first leaf page */
|
| + i64 iLeafRowid;
|
| + Fts5Data *pData;
|
| + int iId = pSeg->pSeg->iSegid;
|
| + u8 aHdr[4] = {0x00, 0x00, 0x00, 0x00};
|
| +
|
| + iLeafRowid = FTS5_SEGMENT_ROWID(iId, pSeg->iTermLeafPgno);
|
| + pData = fts5DataRead(p, iLeafRowid);
|
| + if( pData ){
|
| + fts5BufferZero(&buf);
|
| + fts5BufferGrow(&p->rc, &buf, pData->nn);
|
| + fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr);
|
| + fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n);
|
| + fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p);
|
| + fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff, &pData->p[iOff]);
|
| + if( p->rc==SQLITE_OK ){
|
| + /* Set the szLeaf field */
|
| + fts5PutU16(&buf.p[2], (u16)buf.n);
|
| + }
|
| +
|
| + /* Set up the new page-index array */
|
| + fts5BufferAppendVarint(&p->rc, &buf, 4);
|
| + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno
|
| + && pSeg->iEndofDoclist<pData->szLeaf
|
| + ){
|
| + int nDiff = pData->szLeaf - pSeg->iEndofDoclist;
|
| + fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4);
|
| + fts5BufferAppendBlob(&p->rc, &buf,
|
| + pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff]
|
| + );
|
| + }
|
| +
|
| + fts5DataRelease(pData);
|
| + pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno;
|
| + fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 1), iLeafRowid);
|
| + fts5DataWrite(p, iLeafRowid, buf.p, buf.n);
|
| + }
|
| + }
|
| + }
|
| + fts5BufferFree(&buf);
|
| +}
|
| +
|
| +static void fts5MergeChunkCallback(
|
| + Fts5Index *p,
|
| + void *pCtx,
|
| + const u8 *pChunk, int nChunk
|
| +){
|
| + Fts5SegWriter *pWriter = (Fts5SegWriter*)pCtx;
|
| + fts5WriteAppendPoslistData(p, pWriter, pChunk, nChunk);
|
| +}
|
| +
|
| +/*
|
| +**
|
| +*/
|
| +static void fts5IndexMergeLevel(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */
|
| + int iLvl, /* Level to read input from */
|
| + int *pnRem /* Write up to this many output leaves */
|
| +){
|
| + Fts5Structure *pStruct = *ppStruct;
|
| + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
|
| + Fts5StructureLevel *pLvlOut;
|
| + Fts5IndexIter *pIter = 0; /* Iterator to read input data */
|
| + int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */
|
| + int nInput; /* Number of input segments */
|
| + Fts5SegWriter writer; /* Writer object */
|
| + Fts5StructureSegment *pSeg; /* Output segment */
|
| + Fts5Buffer term;
|
| + int bOldest; /* True if the output segment is the oldest */
|
| +
|
| + assert( iLvl<pStruct->nLevel );
|
| + assert( pLvl->nMerge<=pLvl->nSeg );
|
| +
|
| + memset(&writer, 0, sizeof(Fts5SegWriter));
|
| + memset(&term, 0, sizeof(Fts5Buffer));
|
| + if( pLvl->nMerge ){
|
| + pLvlOut = &pStruct->aLevel[iLvl+1];
|
| + assert( pLvlOut->nSeg>0 );
|
| + nInput = pLvl->nMerge;
|
| + pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1];
|
| +
|
| + fts5WriteInit(p, &writer, pSeg->iSegid);
|
| + writer.writer.pgno = pSeg->pgnoLast+1;
|
| + writer.iBtPage = 0;
|
| + }else{
|
| + int iSegid = fts5AllocateSegid(p, pStruct);
|
| +
|
| + /* Extend the Fts5Structure object as required to ensure the output
|
| + ** segment exists. */
|
| + if( iLvl==pStruct->nLevel-1 ){
|
| + fts5StructureAddLevel(&p->rc, ppStruct);
|
| + pStruct = *ppStruct;
|
| + }
|
| + fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0);
|
| + if( p->rc ) return;
|
| + pLvl = &pStruct->aLevel[iLvl];
|
| + pLvlOut = &pStruct->aLevel[iLvl+1];
|
| +
|
| + fts5WriteInit(p, &writer, iSegid);
|
| +
|
| + /* Add the new segment to the output level */
|
| + pSeg = &pLvlOut->aSeg[pLvlOut->nSeg];
|
| + pLvlOut->nSeg++;
|
| + pSeg->pgnoFirst = 1;
|
| + pSeg->iSegid = iSegid;
|
| + pStruct->nSegment++;
|
| +
|
| + /* Read input from all segments in the input level */
|
| + nInput = pLvl->nSeg;
|
| + }
|
| + bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2);
|
| +
|
| + assert( iLvl>=0 );
|
| + for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, iLvl, nInput, &pIter);
|
| + fts5MultiIterEof(p, pIter)==0;
|
| + fts5MultiIterNext(p, pIter, 0, 0)
|
| + ){
|
| + Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
|
| + int nPos; /* position-list size field value */
|
| + int nTerm;
|
| + const u8 *pTerm;
|
| +
|
| + /* Check for key annihilation. */
|
| + if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue;
|
| +
|
| + pTerm = fts5MultiIterTerm(pIter, &nTerm);
|
| + if( nTerm!=term.n || memcmp(pTerm, term.p, nTerm) ){
|
| + if( pnRem && writer.nLeafWritten>nRem ){
|
| + break;
|
| + }
|
| +
|
| + /* This is a new term. Append a term to the output segment. */
|
| + fts5WriteAppendTerm(p, &writer, nTerm, pTerm);
|
| + fts5BufferSet(&p->rc, &term, nTerm, pTerm);
|
| + }
|
| +
|
| + /* Append the rowid to the output */
|
| + /* WRITEPOSLISTSIZE */
|
| + nPos = pSegIter->nPos*2 + pSegIter->bDel;
|
| + fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter), nPos);
|
| +
|
| + /* Append the position-list data to the output */
|
| + fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback);
|
| + }
|
| +
|
| + /* Flush the last leaf page to disk. Set the output segment b-tree height
|
| + ** and last leaf page number at the same time. */
|
| + fts5WriteFinish(p, &writer, &pSeg->pgnoLast);
|
| +
|
| + if( fts5MultiIterEof(p, pIter) ){
|
| + int i;
|
| +
|
| + /* Remove the redundant segments from the %_data table */
|
| + for(i=0; i<nInput; i++){
|
| + fts5DataRemoveSegment(p, pLvl->aSeg[i].iSegid);
|
| + }
|
| +
|
| + /* Remove the redundant segments from the input level */
|
| + if( pLvl->nSeg!=nInput ){
|
| + int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment);
|
| + memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove);
|
| + }
|
| + pStruct->nSegment -= nInput;
|
| + pLvl->nSeg -= nInput;
|
| + pLvl->nMerge = 0;
|
| + if( pSeg->pgnoLast==0 ){
|
| + pLvlOut->nSeg--;
|
| + pStruct->nSegment--;
|
| + }
|
| + }else{
|
| + assert( pSeg->pgnoLast>0 );
|
| + fts5TrimSegments(p, pIter);
|
| + pLvl->nMerge = nInput;
|
| + }
|
| +
|
| + fts5MultiIterFree(p, pIter);
|
| + fts5BufferFree(&term);
|
| + if( pnRem ) *pnRem -= writer.nLeafWritten;
|
| +}
|
| +
|
| +/*
|
| +** Do up to nPg pages of automerge work on the index.
|
| +*/
|
| +static void fts5IndexMerge(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */
|
| + int nPg /* Pages of work to do */
|
| +){
|
| + int nRem = nPg;
|
| + Fts5Structure *pStruct = *ppStruct;
|
| + while( nRem>0 && p->rc==SQLITE_OK ){
|
| + int iLvl; /* To iterate through levels */
|
| + int iBestLvl = 0; /* Level offering the most input segments */
|
| + int nBest = 0; /* Number of input segments on best level */
|
| +
|
| + /* Set iBestLvl to the level to read input segments from. */
|
| + assert( pStruct->nLevel>0 );
|
| + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
|
| + Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
|
| + if( pLvl->nMerge ){
|
| + if( pLvl->nMerge>nBest ){
|
| + iBestLvl = iLvl;
|
| + nBest = pLvl->nMerge;
|
| + }
|
| + break;
|
| + }
|
| + if( pLvl->nSeg>nBest ){
|
| + nBest = pLvl->nSeg;
|
| + iBestLvl = iLvl;
|
| + }
|
| + }
|
| +
|
| + /* If nBest is still 0, then the index must be empty. */
|
| +#ifdef SQLITE_DEBUG
|
| + for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){
|
| + assert( pStruct->aLevel[iLvl].nSeg==0 );
|
| + }
|
| +#endif
|
| +
|
| + if( nBest<p->pConfig->nAutomerge
|
| + && pStruct->aLevel[iBestLvl].nMerge==0
|
| + ){
|
| + break;
|
| + }
|
| + fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
|
| + if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
|
| + fts5StructurePromote(p, iBestLvl+1, pStruct);
|
| + }
|
| + }
|
| + *ppStruct = pStruct;
|
| +}
|
| +
|
| +/*
|
| +** A total of nLeaf leaf pages of data has just been flushed to a level-0
|
| +** segment. This function updates the write-counter accordingly and, if
|
| +** necessary, performs incremental merge work.
|
| +**
|
| +** If an error occurs, set the Fts5Index.rc error code. If an error has
|
| +** already occurred, this function is a no-op.
|
| +*/
|
| +static void fts5IndexAutomerge(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */
|
| + int nLeaf /* Number of output leaves just written */
|
| +){
|
| + if( p->rc==SQLITE_OK && p->pConfig->nAutomerge>0 ){
|
| + Fts5Structure *pStruct = *ppStruct;
|
| + u64 nWrite; /* Initial value of write-counter */
|
| + int nWork; /* Number of work-quanta to perform */
|
| + int nRem; /* Number of leaf pages left to write */
|
| +
|
| + /* Update the write-counter. While doing so, set nWork. */
|
| + nWrite = pStruct->nWriteCounter;
|
| + nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit));
|
| + pStruct->nWriteCounter += nLeaf;
|
| + nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel);
|
| +
|
| + fts5IndexMerge(p, ppStruct, nRem);
|
| + }
|
| +}
|
| +
|
| +static void fts5IndexCrisismerge(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5Structure **ppStruct /* IN/OUT: Current structure of index */
|
| +){
|
| + const int nCrisis = p->pConfig->nCrisisMerge;
|
| + Fts5Structure *pStruct = *ppStruct;
|
| + int iLvl = 0;
|
| +
|
| + assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 );
|
| + while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){
|
| + fts5IndexMergeLevel(p, &pStruct, iLvl, 0);
|
| + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) );
|
| + fts5StructurePromote(p, iLvl+1, pStruct);
|
| + iLvl++;
|
| + }
|
| + *ppStruct = pStruct;
|
| +}
|
| +
|
| +static int fts5IndexReturn(Fts5Index *p){
|
| + int rc = p->rc;
|
| + p->rc = SQLITE_OK;
|
| + return rc;
|
| +}
|
| +
|
| +typedef struct Fts5FlushCtx Fts5FlushCtx;
|
| +struct Fts5FlushCtx {
|
| + Fts5Index *pIdx;
|
| + Fts5SegWriter writer;
|
| +};
|
| +
|
| +/*
|
| +** Buffer aBuf[] contains a list of varints, all small enough to fit
|
| +** in a 32-bit integer. Return the size of the largest prefix of this
|
| +** list nMax bytes or less in size.
|
| +*/
|
| +static int fts5PoslistPrefix(const u8 *aBuf, int nMax){
|
| + int ret;
|
| + u32 dummy;
|
| + ret = fts5GetVarint32(aBuf, dummy);
|
| + if( ret<nMax ){
|
| + while( 1 ){
|
| + int i = fts5GetVarint32(&aBuf[ret], dummy);
|
| + if( (ret + i) > nMax ) break;
|
| + ret += i;
|
| + }
|
| + }
|
| + return ret;
|
| +}
|
| +
|
| +/*
|
| +** Flush the contents of in-memory hash table iHash to a new level-0
|
| +** segment on disk. Also update the corresponding structure record.
|
| +**
|
| +** If an error occurs, set the Fts5Index.rc error code. If an error has
|
| +** already occurred, this function is a no-op.
|
| +*/
|
| +static void fts5FlushOneHash(Fts5Index *p){
|
| + Fts5Hash *pHash = p->pHash;
|
| + Fts5Structure *pStruct;
|
| + int iSegid;
|
| + int pgnoLast = 0; /* Last leaf page number in segment */
|
| +
|
| + /* Obtain a reference to the index structure and allocate a new segment-id
|
| + ** for the new level-0 segment. */
|
| + pStruct = fts5StructureRead(p);
|
| + iSegid = fts5AllocateSegid(p, pStruct);
|
| +
|
| + if( iSegid ){
|
| + const int pgsz = p->pConfig->pgsz;
|
| +
|
| + Fts5StructureSegment *pSeg; /* New segment within pStruct */
|
| + Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */
|
| + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */
|
| +
|
| + Fts5SegWriter writer;
|
| + fts5WriteInit(p, &writer, iSegid);
|
| +
|
| + pBuf = &writer.writer.buf;
|
| + pPgidx = &writer.writer.pgidx;
|
| +
|
| + /* fts5WriteInit() should have initialized the buffers to (most likely)
|
| + ** the maximum space required. */
|
| + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) );
|
| + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) );
|
| +
|
| + /* Begin scanning through hash table entries. This loop runs once for each
|
| + ** term/doclist currently stored within the hash table. */
|
| + if( p->rc==SQLITE_OK ){
|
| + p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0);
|
| + }
|
| + while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){
|
| + const char *zTerm; /* Buffer containing term */
|
| + const u8 *pDoclist; /* Pointer to doclist for this term */
|
| + int nDoclist; /* Size of doclist in bytes */
|
| +
|
| + /* Write the term for this entry to disk. */
|
| + sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
|
| + fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm);
|
| +
|
| + assert( writer.bFirstRowidInPage==0 );
|
| + if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
|
| + /* The entire doclist will fit on the current leaf. */
|
| + fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
|
| + }else{
|
| + i64 iRowid = 0;
|
| + i64 iDelta = 0;
|
| + int iOff = 0;
|
| +
|
| + /* The entire doclist will not fit on this leaf. The following
|
| + ** loop iterates through the poslists that make up the current
|
| + ** doclist. */
|
| + while( p->rc==SQLITE_OK && iOff<nDoclist ){
|
| + int nPos;
|
| + int nCopy;
|
| + int bDummy;
|
| + iOff += fts5GetVarint(&pDoclist[iOff], (u64*)&iDelta);
|
| + nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
|
| + nCopy += nPos;
|
| + iRowid += iDelta;
|
| +
|
| + if( writer.bFirstRowidInPage ){
|
| + fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */
|
| + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
|
| + writer.bFirstRowidInPage = 0;
|
| + fts5WriteDlidxAppend(p, &writer, iRowid);
|
| + }else{
|
| + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta);
|
| + }
|
| + assert( pBuf->n<=pBuf->nSpace );
|
| +
|
| + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
|
| + /* The entire poslist will fit on the current leaf. So copy
|
| + ** it in one go. */
|
| + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy);
|
| + }else{
|
| + /* The entire poslist will not fit on this leaf. So it needs
|
| + ** to be broken into sections. The only qualification being
|
| + ** that each varint must be stored contiguously. */
|
| + const u8 *pPoslist = &pDoclist[iOff];
|
| + int iPos = 0;
|
| + while( p->rc==SQLITE_OK ){
|
| + int nSpace = pgsz - pBuf->n - pPgidx->n;
|
| + int n = 0;
|
| + if( (nCopy - iPos)<=nSpace ){
|
| + n = nCopy - iPos;
|
| + }else{
|
| + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
|
| + }
|
| + assert( n>0 );
|
| + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
|
| + iPos += n;
|
| + if( (pBuf->n + pPgidx->n)>=pgsz ){
|
| + fts5WriteFlushLeaf(p, &writer);
|
| + }
|
| + if( iPos>=nCopy ) break;
|
| + }
|
| + }
|
| + iOff += nCopy;
|
| + }
|
| + }
|
| +
|
| + /* TODO2: Doclist terminator written here. */
|
| + /* pBuf->p[pBuf->n++] = '\0'; */
|
| + assert( pBuf->n<=pBuf->nSpace );
|
| + sqlite3Fts5HashScanNext(pHash);
|
| + }
|
| + sqlite3Fts5HashClear(pHash);
|
| + fts5WriteFinish(p, &writer, &pgnoLast);
|
| +
|
| + /* Update the Fts5Structure. It is written back to the database by the
|
| + ** fts5StructureRelease() call below. */
|
| + if( pStruct->nLevel==0 ){
|
| + fts5StructureAddLevel(&p->rc, &pStruct);
|
| + }
|
| + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
|
| + if( p->rc==SQLITE_OK ){
|
| + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
|
| + pSeg->iSegid = iSegid;
|
| + pSeg->pgnoFirst = 1;
|
| + pSeg->pgnoLast = pgnoLast;
|
| + pStruct->nSegment++;
|
| + }
|
| + fts5StructurePromote(p, 0, pStruct);
|
| + }
|
| +
|
| + fts5IndexAutomerge(p, &pStruct, pgnoLast);
|
| + fts5IndexCrisismerge(p, &pStruct);
|
| + fts5StructureWrite(p, pStruct);
|
| + fts5StructureRelease(pStruct);
|
| +}
|
| +
|
| +/*
|
| +** Flush any data stored in the in-memory hash tables to the database.
|
| +*/
|
| +static void fts5IndexFlush(Fts5Index *p){
|
| + /* Unless it is empty, flush the hash table to disk */
|
| + if( p->nPendingData ){
|
| + assert( p->pHash );
|
| + p->nPendingData = 0;
|
| + fts5FlushOneHash(p);
|
| + }
|
| +}
|
| +
|
| +
|
| +static int sqlite3Fts5IndexOptimize(Fts5Index *p){
|
| + Fts5Structure *pStruct;
|
| + Fts5Structure *pNew = 0;
|
| + int nSeg = 0;
|
| +
|
| + assert( p->rc==SQLITE_OK );
|
| + fts5IndexFlush(p);
|
| + pStruct = fts5StructureRead(p);
|
| +
|
| + if( pStruct ){
|
| + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
|
| + nSeg = pStruct->nSegment;
|
| + if( nSeg>1 ){
|
| + int nByte = sizeof(Fts5Structure);
|
| + nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
|
| + pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
|
| + }
|
| + }
|
| + if( pNew ){
|
| + Fts5StructureLevel *pLvl;
|
| + int nByte = nSeg * sizeof(Fts5StructureSegment);
|
| + pNew->nLevel = pStruct->nLevel+1;
|
| + pNew->nRef = 1;
|
| + pNew->nWriteCounter = pStruct->nWriteCounter;
|
| + pLvl = &pNew->aLevel[pStruct->nLevel];
|
| + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
|
| + if( pLvl->aSeg ){
|
| + int iLvl, iSeg;
|
| + int iSegOut = 0;
|
| + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
|
| + for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
|
| + pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg];
|
| + iSegOut++;
|
| + }
|
| + }
|
| + pNew->nSegment = pLvl->nSeg = nSeg;
|
| + }else{
|
| + sqlite3_free(pNew);
|
| + pNew = 0;
|
| + }
|
| + }
|
| +
|
| + if( pNew ){
|
| + int iLvl = pNew->nLevel-1;
|
| + while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){
|
| + int nRem = FTS5_OPT_WORK_UNIT;
|
| + fts5IndexMergeLevel(p, &pNew, iLvl, &nRem);
|
| + }
|
| +
|
| + fts5StructureWrite(p, pNew);
|
| + fts5StructureRelease(pNew);
|
| + }
|
| +
|
| + fts5StructureRelease(pStruct);
|
| + return fts5IndexReturn(p);
|
| +}
|
| +
|
| +static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
|
| + Fts5Structure *pStruct;
|
| +
|
| + pStruct = fts5StructureRead(p);
|
| + if( pStruct && pStruct->nLevel ){
|
| + fts5IndexMerge(p, &pStruct, nMerge);
|
| + fts5StructureWrite(p, pStruct);
|
| + }
|
| + fts5StructureRelease(pStruct);
|
| +
|
| + return fts5IndexReturn(p);
|
| +}
|
| +
|
| +static void fts5PoslistCallback(
|
| + Fts5Index *p,
|
| + void *pContext,
|
| + const u8 *pChunk, int nChunk
|
| +){
|
| + assert_nc( nChunk>=0 );
|
| + if( nChunk>0 ){
|
| + fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk);
|
| + }
|
| +}
|
| +
|
| +typedef struct PoslistCallbackCtx PoslistCallbackCtx;
|
| +struct PoslistCallbackCtx {
|
| + Fts5Buffer *pBuf; /* Append to this buffer */
|
| + Fts5Colset *pColset; /* Restrict matches to this column */
|
| + int eState; /* See above */
|
| +};
|
| +
|
| +/*
|
| +** TODO: Make this more efficient!
|
| +*/
|
| +static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){
|
| + int i;
|
| + for(i=0; i<pColset->nCol; i++){
|
| + if( pColset->aiCol[i]==iCol ) return 1;
|
| + }
|
| + return 0;
|
| +}
|
| +
|
| +static void fts5PoslistFilterCallback(
|
| + Fts5Index *p,
|
| + void *pContext,
|
| + const u8 *pChunk, int nChunk
|
| +){
|
| + PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext;
|
| + assert_nc( nChunk>=0 );
|
| + if( nChunk>0 ){
|
| + /* Search through to find the first varint with value 1. This is the
|
| + ** start of the next columns hits. */
|
| + int i = 0;
|
| + int iStart = 0;
|
| +
|
| + if( pCtx->eState==2 ){
|
| + int iCol;
|
| + fts5FastGetVarint32(pChunk, i, iCol);
|
| + if( fts5IndexColsetTest(pCtx->pColset, iCol) ){
|
| + pCtx->eState = 1;
|
| + fts5BufferSafeAppendVarint(pCtx->pBuf, 1);
|
| + }else{
|
| + pCtx->eState = 0;
|
| + }
|
| + }
|
| +
|
| + do {
|
| + while( i<nChunk && pChunk[i]!=0x01 ){
|
| + while( pChunk[i] & 0x80 ) i++;
|
| + i++;
|
| + }
|
| + if( pCtx->eState ){
|
| + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart);
|
| + }
|
| + if( i<nChunk ){
|
| + int iCol;
|
| + iStart = i;
|
| + i++;
|
| + if( i>=nChunk ){
|
| + pCtx->eState = 2;
|
| + }else{
|
| + fts5FastGetVarint32(pChunk, i, iCol);
|
| + pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol);
|
| + if( pCtx->eState ){
|
| + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart);
|
| + iStart = i;
|
| + }
|
| + }
|
| + }
|
| + }while( i<nChunk );
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Iterator pIter currently points to a valid entry (not EOF). This
|
| +** function appends the position list data for the current entry to
|
| +** buffer pBuf. It does not make a copy of the position-list size
|
| +** field.
|
| +*/
|
| +static void fts5SegiterPoslist(
|
| + Fts5Index *p,
|
| + Fts5SegIter *pSeg,
|
| + Fts5Colset *pColset,
|
| + Fts5Buffer *pBuf
|
| +){
|
| + if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){
|
| + if( pColset==0 ){
|
| + fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
|
| + }else{
|
| + PoslistCallbackCtx sCtx;
|
| + sCtx.pBuf = pBuf;
|
| + sCtx.pColset = pColset;
|
| + sCtx.eState = fts5IndexColsetTest(pColset, 0);
|
| + assert( sCtx.eState==0 || sCtx.eState==1 );
|
| + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** IN/OUT parameter (*pa) points to a position list n bytes in size. If
|
| +** the position list contains entries for column iCol, then (*pa) is set
|
| +** to point to the sub-position-list for that column and the number of
|
| +** bytes in it returned. Or, if the argument position list does not
|
| +** contain any entries for column iCol, return 0.
|
| +*/
|
| +static int fts5IndexExtractCol(
|
| + const u8 **pa, /* IN/OUT: Pointer to poslist */
|
| + int n, /* IN: Size of poslist in bytes */
|
| + int iCol /* Column to extract from poslist */
|
| +){
|
| + int iCurrent = 0; /* Anything before the first 0x01 is col 0 */
|
| + const u8 *p = *pa;
|
| + const u8 *pEnd = &p[n]; /* One byte past end of position list */
|
| + u8 prev = 0;
|
| +
|
| + while( iCol>iCurrent ){
|
| + /* Advance pointer p until it points to pEnd or an 0x01 byte that is
|
| + ** not part of a varint */
|
| + while( (prev & 0x80) || *p!=0x01 ){
|
| + prev = *p++;
|
| + if( p==pEnd ) return 0;
|
| + }
|
| + *pa = p++;
|
| + p += fts5GetVarint32(p, iCurrent);
|
| + }
|
| + if( iCol!=iCurrent ) return 0;
|
| +
|
| + /* Advance pointer p until it points to pEnd or an 0x01 byte that is
|
| + ** not part of a varint */
|
| + assert( (prev & 0x80)==0 );
|
| + while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){
|
| + prev = *p++;
|
| + }
|
| + return p - (*pa);
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Iterator pMulti currently points to a valid entry (not EOF). This
|
| +** function appends the following to buffer pBuf:
|
| +**
|
| +** * The varint iDelta, and
|
| +** * the position list that currently points to, including the size field.
|
| +**
|
| +** If argument pColset is NULL, then the position list is filtered according
|
| +** to pColset before being appended to the buffer. If this means there are
|
| +** no entries in the position list, nothing is appended to the buffer (not
|
| +** even iDelta).
|
| +**
|
| +** If an error occurs, an error code is left in p->rc.
|
| +*/
|
| +static int fts5AppendPoslist(
|
| + Fts5Index *p,
|
| + i64 iDelta,
|
| + Fts5IndexIter *pMulti,
|
| + Fts5Colset *pColset,
|
| + Fts5Buffer *pBuf
|
| +){
|
| + if( p->rc==SQLITE_OK ){
|
| + Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
|
| + assert( fts5MultiIterEof(p, pMulti)==0 );
|
| + assert( pSeg->nPos>0 );
|
| + if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){
|
| +
|
| + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf
|
| + && (pColset==0 || pColset->nCol==1)
|
| + ){
|
| + const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
|
| + int nPos;
|
| + if( pColset ){
|
| + nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]);
|
| + if( nPos==0 ) return 1;
|
| + }else{
|
| + nPos = pSeg->nPos;
|
| + }
|
| + assert( nPos>0 );
|
| + fts5BufferSafeAppendVarint(pBuf, iDelta);
|
| + fts5BufferSafeAppendVarint(pBuf, nPos*2);
|
| + fts5BufferSafeAppendBlob(pBuf, pPos, nPos);
|
| + }else{
|
| + int iSv1;
|
| + int iSv2;
|
| + int iData;
|
| +
|
| + /* Append iDelta */
|
| + iSv1 = pBuf->n;
|
| + fts5BufferSafeAppendVarint(pBuf, iDelta);
|
| +
|
| + /* WRITEPOSLISTSIZE */
|
| + iSv2 = pBuf->n;
|
| + fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2);
|
| + iData = pBuf->n;
|
| +
|
| + fts5SegiterPoslist(p, pSeg, pColset, pBuf);
|
| +
|
| + if( pColset ){
|
| + int nActual = pBuf->n - iData;
|
| + if( nActual!=pSeg->nPos ){
|
| + if( nActual==0 ){
|
| + pBuf->n = iSv1;
|
| + return 1;
|
| + }else{
|
| + int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
|
| + while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; }
|
| + sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2);
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + }
|
| + }
|
| +
|
| + return 0;
|
| +}
|
| +
|
| +static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
|
| + u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;
|
| +
|
| + assert( pIter->aPoslist );
|
| + if( p>=pIter->aEof ){
|
| + pIter->aPoslist = 0;
|
| + }else{
|
| + i64 iDelta;
|
| +
|
| + p += fts5GetVarint(p, (u64*)&iDelta);
|
| + pIter->iRowid += iDelta;
|
| +
|
| + /* Read position list size */
|
| + if( p[0] & 0x80 ){
|
| + int nPos;
|
| + pIter->nSize = fts5GetVarint32(p, nPos);
|
| + pIter->nPoslist = (nPos>>1);
|
| + }else{
|
| + pIter->nPoslist = ((int)(p[0])) >> 1;
|
| + pIter->nSize = 1;
|
| + }
|
| +
|
| + pIter->aPoslist = p;
|
| + }
|
| +}
|
| +
|
| +static void fts5DoclistIterInit(
|
| + Fts5Buffer *pBuf,
|
| + Fts5DoclistIter *pIter
|
| +){
|
| + memset(pIter, 0, sizeof(*pIter));
|
| + pIter->aPoslist = pBuf->p;
|
| + pIter->aEof = &pBuf->p[pBuf->n];
|
| + fts5DoclistIterNext(pIter);
|
| +}
|
| +
|
| +#if 0
|
| +/*
|
| +** Append a doclist to buffer pBuf.
|
| +**
|
| +** This function assumes that space within the buffer has already been
|
| +** allocated.
|
| +*/
|
| +static void fts5MergeAppendDocid(
|
| + Fts5Buffer *pBuf, /* Buffer to write to */
|
| + i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */
|
| + i64 iRowid /* Rowid to append */
|
| +){
|
| + assert( pBuf->n!=0 || (*piLastRowid)==0 );
|
| + fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid);
|
| + *piLastRowid = iRowid;
|
| +}
|
| +#endif
|
| +
|
| +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \
|
| + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \
|
| + fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \
|
| + (iLastRowid) = (iRowid); \
|
| +}
|
| +
|
| +/*
|
| +** Buffers p1 and p2 contain doclists. This function merges the content
|
| +** of the two doclists together and sets buffer p1 to the result before
|
| +** returning.
|
| +**
|
| +** If an error occurs, an error code is left in p->rc. If an error has
|
| +** already occurred, this function is a no-op.
|
| +*/
|
| +static void fts5MergePrefixLists(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5Buffer *p1, /* First list to merge */
|
| + Fts5Buffer *p2 /* Second list to merge */
|
| +){
|
| + if( p2->n ){
|
| + i64 iLastRowid = 0;
|
| + Fts5DoclistIter i1;
|
| + Fts5DoclistIter i2;
|
| + Fts5Buffer out;
|
| + Fts5Buffer tmp;
|
| + memset(&out, 0, sizeof(out));
|
| + memset(&tmp, 0, sizeof(tmp));
|
| +
|
| + sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n);
|
| + fts5DoclistIterInit(p1, &i1);
|
| + fts5DoclistIterInit(p2, &i2);
|
| + while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){
|
| + if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowid<i2.iRowid) ){
|
| + /* Copy entry from i1 */
|
| + fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
|
| + fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize);
|
| + fts5DoclistIterNext(&i1);
|
| + }
|
| + else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
|
| + /* Copy entry from i2 */
|
| + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
|
| + fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize);
|
| + fts5DoclistIterNext(&i2);
|
| + }
|
| + else{
|
| + i64 iPos1 = 0;
|
| + i64 iPos2 = 0;
|
| + int iOff1 = 0;
|
| + int iOff2 = 0;
|
| + u8 *a1 = &i1.aPoslist[i1.nSize];
|
| + u8 *a2 = &i2.aPoslist[i2.nSize];
|
| +
|
| + Fts5PoslistWriter writer;
|
| + memset(&writer, 0, sizeof(writer));
|
| +
|
| + /* Merge the two position lists. */
|
| + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
|
| + fts5BufferZero(&tmp);
|
| +
|
| + sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
|
| + sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
|
| +
|
| + while( p->rc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){
|
| + i64 iNew;
|
| + if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){
|
| + iNew = iPos1;
|
| + sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
|
| + }else{
|
| + iNew = iPos2;
|
| + sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
|
| + if( iPos1==iPos2 ){
|
| + sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1);
|
| + }
|
| + }
|
| + p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
|
| + }
|
| +
|
| + /* WRITEPOSLISTSIZE */
|
| + fts5BufferSafeAppendVarint(&out, tmp.n * 2);
|
| + fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n);
|
| + fts5DoclistIterNext(&i1);
|
| + fts5DoclistIterNext(&i2);
|
| + }
|
| + }
|
| +
|
| + fts5BufferSet(&p->rc, p1, out.n, out.p);
|
| + fts5BufferFree(&tmp);
|
| + fts5BufferFree(&out);
|
| + }
|
| +}
|
| +
|
| +static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
|
| + Fts5Buffer tmp = *p1;
|
| + *p1 = *p2;
|
| + *p2 = tmp;
|
| +}
|
| +
|
| +static void fts5SetupPrefixIter(
|
| + Fts5Index *p, /* Index to read from */
|
| + int bDesc, /* True for "ORDER BY rowid DESC" */
|
| + const u8 *pToken, /* Buffer containing prefix to match */
|
| + int nToken, /* Size of buffer pToken in bytes */
|
| + Fts5Colset *pColset, /* Restrict matches to these columns */
|
| + Fts5IndexIter **ppIter /* OUT: New iterator */
|
| +){
|
| + Fts5Structure *pStruct;
|
| + Fts5Buffer *aBuf;
|
| + const int nBuf = 32;
|
| +
|
| + aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
|
| + pStruct = fts5StructureRead(p);
|
| +
|
| + if( aBuf && pStruct ){
|
| + const int flags = FTS5INDEX_QUERY_SCAN;
|
| + int i;
|
| + i64 iLastRowid = 0;
|
| + Fts5IndexIter *p1 = 0; /* Iterator used to gather data from index */
|
| + Fts5Data *pData;
|
| + Fts5Buffer doclist;
|
| + int bNewTerm = 1;
|
| +
|
| + memset(&doclist, 0, sizeof(doclist));
|
| + for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1);
|
| + fts5MultiIterEof(p, p1)==0;
|
| + fts5MultiIterNext2(p, p1, &bNewTerm)
|
| + ){
|
| + i64 iRowid = fts5MultiIterRowid(p1);
|
| + int nTerm;
|
| + const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
|
| + assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
|
| + if( bNewTerm ){
|
| + if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;
|
| + }
|
| +
|
| + if( doclist.n>0 && iRowid<=iLastRowid ){
|
| + for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
|
| + assert( i<nBuf );
|
| + if( aBuf[i].n==0 ){
|
| + fts5BufferSwap(&doclist, &aBuf[i]);
|
| + fts5BufferZero(&doclist);
|
| + }else{
|
| + fts5MergePrefixLists(p, &doclist, &aBuf[i]);
|
| + fts5BufferZero(&aBuf[i]);
|
| + }
|
| + }
|
| + iLastRowid = 0;
|
| + }
|
| +
|
| + if( !fts5AppendPoslist(p, iRowid-iLastRowid, p1, pColset, &doclist) ){
|
| + iLastRowid = iRowid;
|
| + }
|
| + }
|
| +
|
| + for(i=0; i<nBuf; i++){
|
| + if( p->rc==SQLITE_OK ){
|
| + fts5MergePrefixLists(p, &doclist, &aBuf[i]);
|
| + }
|
| + fts5BufferFree(&aBuf[i]);
|
| + }
|
| + fts5MultiIterFree(p, p1);
|
| +
|
| + pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n);
|
| + if( pData ){
|
| + pData->p = (u8*)&pData[1];
|
| + pData->nn = pData->szLeaf = doclist.n;
|
| + memcpy(pData->p, doclist.p, doclist.n);
|
| + fts5MultiIterNew2(p, pData, bDesc, ppIter);
|
| + }
|
| + fts5BufferFree(&doclist);
|
| + }
|
| +
|
| + fts5StructureRelease(pStruct);
|
| + sqlite3_free(aBuf);
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
|
| +** to the document with rowid iRowid.
|
| +*/
|
| +static int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
|
| + assert( p->rc==SQLITE_OK );
|
| +
|
| + /* Allocate the hash table if it has not already been allocated */
|
| + if( p->pHash==0 ){
|
| + p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);
|
| + }
|
| +
|
| + /* Flush the hash table to disk if required */
|
| + if( iRowid<p->iWriteRowid
|
| + || (iRowid==p->iWriteRowid && p->bDelete==0)
|
| + || (p->nPendingData > p->pConfig->nHashSize)
|
| + ){
|
| + fts5IndexFlush(p);
|
| + }
|
| +
|
| + p->iWriteRowid = iRowid;
|
| + p->bDelete = bDelete;
|
| + return fts5IndexReturn(p);
|
| +}
|
| +
|
| +/*
|
| +** Commit data to disk.
|
| +*/
|
| +static int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
|
| + assert( p->rc==SQLITE_OK );
|
| + fts5IndexFlush(p);
|
| + if( bCommit ) fts5CloseReader(p);
|
| + return fts5IndexReturn(p);
|
| +}
|
| +
|
| +/*
|
| +** Discard any data stored in the in-memory hash tables. Do not write it
|
| +** to the database. Additionally, assume that the contents of the %_data
|
| +** table may have changed on disk. So any in-memory caches of %_data
|
| +** records must be invalidated.
|
| +*/
|
| +static int sqlite3Fts5IndexRollback(Fts5Index *p){
|
| + fts5CloseReader(p);
|
| + fts5IndexDiscardData(p);
|
| + assert( p->rc==SQLITE_OK );
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** The %_data table is completely empty when this function is called. This
|
| +** function populates it with the initial structure objects for each index,
|
| +** and the initial version of the "averages" record (a zero-byte blob).
|
| +*/
|
| +static int sqlite3Fts5IndexReinit(Fts5Index *p){
|
| + Fts5Structure s;
|
| + memset(&s, 0, sizeof(Fts5Structure));
|
| + fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
|
| + fts5StructureWrite(p, &s);
|
| + return fts5IndexReturn(p);
|
| +}
|
| +
|
| +/*
|
| +** Open a new Fts5Index handle. If the bCreate argument is true, create
|
| +** and initialize the underlying %_data table.
|
| +**
|
| +** If successful, set *pp to point to the new object and return SQLITE_OK.
|
| +** Otherwise, set *pp to NULL and return an SQLite error code.
|
| +*/
|
| +static int sqlite3Fts5IndexOpen(
|
| + Fts5Config *pConfig,
|
| + int bCreate,
|
| + Fts5Index **pp,
|
| + char **pzErr
|
| +){
|
| + int rc = SQLITE_OK;
|
| + Fts5Index *p; /* New object */
|
| +
|
| + *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index));
|
| + if( rc==SQLITE_OK ){
|
| + p->pConfig = pConfig;
|
| + p->nWorkUnit = FTS5_WORK_UNIT;
|
| + p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName);
|
| + if( p->zDataTbl && bCreate ){
|
| + rc = sqlite3Fts5CreateTable(
|
| + pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
|
| + );
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5CreateTable(pConfig, "idx",
|
| + "segid, term, pgno, PRIMARY KEY(segid, term)",
|
| + 1, pzErr
|
| + );
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5IndexReinit(p);
|
| + }
|
| + }
|
| + }
|
| +
|
| + assert( rc!=SQLITE_OK || p->rc==SQLITE_OK );
|
| + if( rc ){
|
| + sqlite3Fts5IndexClose(p);
|
| + *pp = 0;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
|
| +*/
|
| +static int sqlite3Fts5IndexClose(Fts5Index *p){
|
| + int rc = SQLITE_OK;
|
| + if( p ){
|
| + assert( p->pReader==0 );
|
| + sqlite3_finalize(p->pWriter);
|
| + sqlite3_finalize(p->pDeleter);
|
| + sqlite3_finalize(p->pIdxWriter);
|
| + sqlite3_finalize(p->pIdxDeleter);
|
| + sqlite3_finalize(p->pIdxSelect);
|
| + sqlite3Fts5HashFree(p->pHash);
|
| + sqlite3_free(p->zDataTbl);
|
| + sqlite3_free(p);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Argument p points to a buffer containing utf-8 text that is n bytes in
|
| +** size. Return the number of bytes in the nChar character prefix of the
|
| +** buffer, or 0 if there are less than nChar characters in total.
|
| +*/
|
| +static int fts5IndexCharlenToBytelen(const char *p, int nByte, int nChar){
|
| + int n = 0;
|
| + int i;
|
| + for(i=0; i<nChar; i++){
|
| + if( n>=nByte ) return 0; /* Input contains fewer than nChar chars */
|
| + if( (unsigned char)p[n++]>=0xc0 ){
|
| + while( (p[n] & 0xc0)==0x80 ) n++;
|
| + }
|
| + }
|
| + return n;
|
| +}
|
| +
|
| +/*
|
| +** pIn is a UTF-8 encoded string, nIn bytes in size. Return the number of
|
| +** unicode characters in the string.
|
| +*/
|
| +static int fts5IndexCharlen(const char *pIn, int nIn){
|
| + int nChar = 0;
|
| + int i = 0;
|
| + while( i<nIn ){
|
| + if( (unsigned char)pIn[i++]>=0xc0 ){
|
| + while( i<nIn && (pIn[i] & 0xc0)==0x80 ) i++;
|
| + }
|
| + nChar++;
|
| + }
|
| + return nChar;
|
| +}
|
| +
|
| +/*
|
| +** Insert or remove data to or from the index. Each time a document is
|
| +** added to or removed from the index, this function is called one or more
|
| +** times.
|
| +**
|
| +** For an insert, it must be called once for each token in the new document.
|
| +** If the operation is a delete, it must be called (at least) once for each
|
| +** unique token in the document with an iCol value less than zero. The iPos
|
| +** argument is ignored for a delete.
|
| +*/
|
| +static int sqlite3Fts5IndexWrite(
|
| + Fts5Index *p, /* Index to write to */
|
| + int iCol, /* Column token appears in (-ve -> delete) */
|
| + int iPos, /* Position of token within column */
|
| + const char *pToken, int nToken /* Token to add or remove to or from index */
|
| +){
|
| + int i; /* Used to iterate through indexes */
|
| + int rc = SQLITE_OK; /* Return code */
|
| + Fts5Config *pConfig = p->pConfig;
|
| +
|
| + assert( p->rc==SQLITE_OK );
|
| + assert( (iCol<0)==p->bDelete );
|
| +
|
| + /* Add the entry to the main terms index. */
|
| + rc = sqlite3Fts5HashWrite(
|
| + p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken
|
| + );
|
| +
|
| + for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){
|
| + int nByte = fts5IndexCharlenToBytelen(pToken, nToken, pConfig->aPrefix[i]);
|
| + if( nByte ){
|
| + rc = sqlite3Fts5HashWrite(p->pHash,
|
| + p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken,
|
| + nByte
|
| + );
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Open a new iterator to iterate though all rowid that match the
|
| +** specified token or token prefix.
|
| +*/
|
| +static int sqlite3Fts5IndexQuery(
|
| + Fts5Index *p, /* FTS index to query */
|
| + const char *pToken, int nToken, /* Token (or prefix) to query for */
|
| + int flags, /* Mask of FTS5INDEX_QUERY_X flags */
|
| + Fts5Colset *pColset, /* Match these columns only */
|
| + Fts5IndexIter **ppIter /* OUT: New iterator object */
|
| +){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + Fts5IndexIter *pRet = 0;
|
| + int iIdx = 0;
|
| + Fts5Buffer buf = {0, 0, 0};
|
| +
|
| + /* If the QUERY_SCAN flag is set, all other flags must be clear. */
|
| + assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN );
|
| +
|
| + if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
|
| + memcpy(&buf.p[1], pToken, nToken);
|
| +
|
| +#ifdef SQLITE_DEBUG
|
| + /* If the QUERY_TEST_NOIDX flag was specified, then this must be a
|
| + ** prefix-query. Instead of using a prefix-index (if one exists),
|
| + ** evaluate the prefix query using the main FTS index. This is used
|
| + ** for internal sanity checking by the integrity-check in debug
|
| + ** mode only. */
|
| + if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){
|
| + assert( flags & FTS5INDEX_QUERY_PREFIX );
|
| + iIdx = 1+pConfig->nPrefix;
|
| + }else
|
| +#endif
|
| + if( flags & FTS5INDEX_QUERY_PREFIX ){
|
| + int nChar = fts5IndexCharlen(pToken, nToken);
|
| + for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
|
| + if( pConfig->aPrefix[iIdx-1]==nChar ) break;
|
| + }
|
| + }
|
| +
|
| + if( iIdx<=pConfig->nPrefix ){
|
| + Fts5Structure *pStruct = fts5StructureRead(p);
|
| + buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx);
|
| + if( pStruct ){
|
| + fts5MultiIterNew(p, pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet);
|
| + fts5StructureRelease(pStruct);
|
| + }
|
| + }else{
|
| + int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
|
| + buf.p[0] = FTS5_MAIN_PREFIX;
|
| + fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet);
|
| + }
|
| +
|
| + if( p->rc ){
|
| + sqlite3Fts5IterClose(pRet);
|
| + pRet = 0;
|
| + fts5CloseReader(p);
|
| + }
|
| + *ppIter = pRet;
|
| + sqlite3Fts5BufferFree(&buf);
|
| + }
|
| + return fts5IndexReturn(p);
|
| +}
|
| +
|
| +/*
|
| +** Return true if the iterator passed as the only argument is at EOF.
|
| +*/
|
| +static int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
|
| + assert( pIter->pIndex->rc==SQLITE_OK );
|
| + return pIter->bEof;
|
| +}
|
| +
|
| +/*
|
| +** Move to the next matching rowid.
|
| +*/
|
| +static int sqlite3Fts5IterNext(Fts5IndexIter *pIter){
|
| + assert( pIter->pIndex->rc==SQLITE_OK );
|
| + fts5MultiIterNext(pIter->pIndex, pIter, 0, 0);
|
| + return fts5IndexReturn(pIter->pIndex);
|
| +}
|
| +
|
| +/*
|
| +** Move to the next matching term/rowid. Used by the fts5vocab module.
|
| +*/
|
| +static int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){
|
| + Fts5Index *p = pIter->pIndex;
|
| +
|
| + assert( pIter->pIndex->rc==SQLITE_OK );
|
| +
|
| + fts5MultiIterNext(p, pIter, 0, 0);
|
| + if( p->rc==SQLITE_OK ){
|
| + Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
|
| + if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){
|
| + fts5DataRelease(pSeg->pLeaf);
|
| + pSeg->pLeaf = 0;
|
| + pIter->bEof = 1;
|
| + }
|
| + }
|
| +
|
| + return fts5IndexReturn(pIter->pIndex);
|
| +}
|
| +
|
| +/*
|
| +** Move to the next matching rowid that occurs at or after iMatch. The
|
| +** definition of "at or after" depends on whether this iterator iterates
|
| +** in ascending or descending rowid order.
|
| +*/
|
| +static int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
|
| + fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch);
|
| + return fts5IndexReturn(pIter->pIndex);
|
| +}
|
| +
|
| +/*
|
| +** Return the current rowid.
|
| +*/
|
| +static i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
|
| + return fts5MultiIterRowid(pIter);
|
| +}
|
| +
|
| +/*
|
| +** Return the current term.
|
| +*/
|
| +static const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){
|
| + int n;
|
| + const char *z = (const char*)fts5MultiIterTerm(pIter, &n);
|
| + *pn = n-1;
|
| + return &z[1];
|
| +}
|
| +
|
| +
|
| +static int fts5IndexExtractColset (
|
| + Fts5Colset *pColset, /* Colset to filter on */
|
| + const u8 *pPos, int nPos, /* Position list */
|
| + Fts5Buffer *pBuf /* Output buffer */
|
| +){
|
| + int rc = SQLITE_OK;
|
| + int i;
|
| +
|
| + fts5BufferZero(pBuf);
|
| + for(i=0; i<pColset->nCol; i++){
|
| + const u8 *pSub = pPos;
|
| + int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
|
| + if( nSub ){
|
| + fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Return a pointer to a buffer containing a copy of the position list for
|
| +** the current entry. Output variable *pn is set to the size of the buffer
|
| +** in bytes before returning.
|
| +**
|
| +** The returned position list does not include the "number of bytes" varint
|
| +** field that starts the position list on disk.
|
| +*/
|
| +static int sqlite3Fts5IterPoslist(
|
| + Fts5IndexIter *pIter,
|
| + Fts5Colset *pColset, /* Column filter (or NULL) */
|
| + const u8 **pp, /* OUT: Pointer to position-list data */
|
| + int *pn, /* OUT: Size of position-list in bytes */
|
| + i64 *piRowid /* OUT: Current rowid */
|
| +){
|
| + Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
|
| + assert( pIter->pIndex->rc==SQLITE_OK );
|
| + *piRowid = pSeg->iRowid;
|
| + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
|
| + u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
|
| + if( pColset==0 || pIter->bFiltered ){
|
| + *pn = pSeg->nPos;
|
| + *pp = pPos;
|
| + }else if( pColset->nCol==1 ){
|
| + *pp = pPos;
|
| + *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]);
|
| + }else{
|
| + fts5BufferZero(&pIter->poslist);
|
| + fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist);
|
| + *pp = pIter->poslist.p;
|
| + *pn = pIter->poslist.n;
|
| + }
|
| + }else{
|
| + fts5BufferZero(&pIter->poslist);
|
| + fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
|
| + *pp = pIter->poslist.p;
|
| + *pn = pIter->poslist.n;
|
| + }
|
| + return fts5IndexReturn(pIter->pIndex);
|
| +}
|
| +
|
| +/*
|
| +** This function is similar to sqlite3Fts5IterPoslist(), except that it
|
| +** copies the position list into the buffer supplied as the second
|
| +** argument.
|
| +*/
|
| +static int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
|
| + Fts5Index *p = pIter->pIndex;
|
| + Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
|
| + assert( p->rc==SQLITE_OK );
|
| + fts5BufferZero(pBuf);
|
| + fts5SegiterPoslist(p, pSeg, 0, pBuf);
|
| + return fts5IndexReturn(p);
|
| +}
|
| +
|
| +/*
|
| +** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
|
| +*/
|
| +static void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
|
| + if( pIter ){
|
| + Fts5Index *pIndex = pIter->pIndex;
|
| + fts5MultiIterFree(pIter->pIndex, pIter);
|
| + fts5CloseReader(pIndex);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Read and decode the "averages" record from the database.
|
| +**
|
| +** Parameter anSize must point to an array of size nCol, where nCol is
|
| +** the number of user defined columns in the FTS table.
|
| +*/
|
| +static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){
|
| + int nCol = p->pConfig->nCol;
|
| + Fts5Data *pData;
|
| +
|
| + *pnRow = 0;
|
| + memset(anSize, 0, sizeof(i64) * nCol);
|
| + pData = fts5DataRead(p, FTS5_AVERAGES_ROWID);
|
| + if( p->rc==SQLITE_OK && pData->nn ){
|
| + int i = 0;
|
| + int iCol;
|
| + i += fts5GetVarint(&pData->p[i], (u64*)pnRow);
|
| + for(iCol=0; i<pData->nn && iCol<nCol; iCol++){
|
| + i += fts5GetVarint(&pData->p[i], (u64*)&anSize[iCol]);
|
| + }
|
| + }
|
| +
|
| + fts5DataRelease(pData);
|
| + return fts5IndexReturn(p);
|
| +}
|
| +
|
| +/*
|
| +** Replace the current "averages" record with the contents of the buffer
|
| +** supplied as the second argument.
|
| +*/
|
| +static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){
|
| + assert( p->rc==SQLITE_OK );
|
| + fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData);
|
| + return fts5IndexReturn(p);
|
| +}
|
| +
|
| +/*
|
| +** Return the total number of blocks this module has read from the %_data
|
| +** table since it was created.
|
| +*/
|
| +static int sqlite3Fts5IndexReads(Fts5Index *p){
|
| + return p->nRead;
|
| +}
|
| +
|
| +/*
|
| +** Set the 32-bit cookie value stored at the start of all structure
|
| +** records to the value passed as the second argument.
|
| +**
|
| +** Return SQLITE_OK if successful, or an SQLite error code if an error
|
| +** occurs.
|
| +*/
|
| +static int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){
|
| + int rc; /* Return code */
|
| + Fts5Config *pConfig = p->pConfig; /* Configuration object */
|
| + u8 aCookie[4]; /* Binary representation of iNew */
|
| + sqlite3_blob *pBlob = 0;
|
| +
|
| + assert( p->rc==SQLITE_OK );
|
| + sqlite3Fts5Put32(aCookie, iNew);
|
| +
|
| + rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl,
|
| + "block", FTS5_STRUCTURE_ROWID, 1, &pBlob
|
| + );
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_blob_write(pBlob, aCookie, 4, 0);
|
| + rc = sqlite3_blob_close(pBlob);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +static int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
|
| + Fts5Structure *pStruct;
|
| + pStruct = fts5StructureRead(p);
|
| + fts5StructureRelease(pStruct);
|
| + return fts5IndexReturn(p);
|
| +}
|
| +
|
| +
|
| +/*************************************************************************
|
| +**************************************************************************
|
| +** Below this point is the implementation of the integrity-check
|
| +** functionality.
|
| +*/
|
| +
|
| +/*
|
| +** Return a simple checksum value based on the arguments.
|
| +*/
|
| +static u64 fts5IndexEntryCksum(
|
| + i64 iRowid,
|
| + int iCol,
|
| + int iPos,
|
| + int iIdx,
|
| + const char *pTerm,
|
| + int nTerm
|
| +){
|
| + int i;
|
| + u64 ret = iRowid;
|
| + ret += (ret<<3) + iCol;
|
| + ret += (ret<<3) + iPos;
|
| + if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX + iIdx);
|
| + for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i];
|
| + return ret;
|
| +}
|
| +
|
| +#ifdef SQLITE_DEBUG
|
| +/*
|
| +** This function is purely an internal test. It does not contribute to
|
| +** FTS functionality, or even the integrity-check, in any way.
|
| +**
|
| +** Instead, it tests that the same set of pgno/rowid combinations are
|
| +** visited regardless of whether the doclist-index identified by parameters
|
| +** iSegid/iLeaf is iterated in forwards or reverse order.
|
| +*/
|
| +static void fts5TestDlidxReverse(
|
| + Fts5Index *p,
|
| + int iSegid, /* Segment id to load from */
|
| + int iLeaf /* Load doclist-index for this leaf */
|
| +){
|
| + Fts5DlidxIter *pDlidx = 0;
|
| + u64 cksum1 = 13;
|
| + u64 cksum2 = 13;
|
| +
|
| + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iLeaf);
|
| + fts5DlidxIterEof(p, pDlidx)==0;
|
| + fts5DlidxIterNext(p, pDlidx)
|
| + ){
|
| + i64 iRowid = fts5DlidxIterRowid(pDlidx);
|
| + int pgno = fts5DlidxIterPgno(pDlidx);
|
| + assert( pgno>iLeaf );
|
| + cksum1 += iRowid + ((i64)pgno<<32);
|
| + }
|
| + fts5DlidxIterFree(pDlidx);
|
| + pDlidx = 0;
|
| +
|
| + for(pDlidx=fts5DlidxIterInit(p, 1, iSegid, iLeaf);
|
| + fts5DlidxIterEof(p, pDlidx)==0;
|
| + fts5DlidxIterPrev(p, pDlidx)
|
| + ){
|
| + i64 iRowid = fts5DlidxIterRowid(pDlidx);
|
| + int pgno = fts5DlidxIterPgno(pDlidx);
|
| + assert( fts5DlidxIterPgno(pDlidx)>iLeaf );
|
| + cksum2 += iRowid + ((i64)pgno<<32);
|
| + }
|
| + fts5DlidxIterFree(pDlidx);
|
| + pDlidx = 0;
|
| +
|
| + if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT;
|
| +}
|
| +
|
| +static int fts5QueryCksum(
|
| + Fts5Index *p, /* Fts5 index object */
|
| + int iIdx,
|
| + const char *z, /* Index key to query for */
|
| + int n, /* Size of index key in bytes */
|
| + int flags, /* Flags for Fts5IndexQuery */
|
| + u64 *pCksum /* IN/OUT: Checksum value */
|
| +){
|
| + u64 cksum = *pCksum;
|
| + Fts5IndexIter *pIdxIter = 0;
|
| + int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);
|
| +
|
| + while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
|
| + i64 dummy;
|
| + const u8 *pPos;
|
| + int nPos;
|
| + i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
|
| + rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy);
|
| + if( rc==SQLITE_OK ){
|
| + Fts5PoslistReader sReader;
|
| + for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader);
|
| + sReader.bEof==0;
|
| + sqlite3Fts5PoslistReaderNext(&sReader)
|
| + ){
|
| + int iCol = FTS5_POS2COLUMN(sReader.iPos);
|
| + int iOff = FTS5_POS2OFFSET(sReader.iPos);
|
| + cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
|
| + }
|
| + rc = sqlite3Fts5IterNext(pIdxIter);
|
| + }
|
| + }
|
| + sqlite3Fts5IterClose(pIdxIter);
|
| +
|
| + *pCksum = cksum;
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** This function is also purely an internal test. It does not contribute to
|
| +** FTS functionality, or even the integrity-check, in any way.
|
| +*/
|
| +static void fts5TestTerm(
|
| + Fts5Index *p,
|
| + Fts5Buffer *pPrev, /* Previous term */
|
| + const char *z, int n, /* Possibly new term to test */
|
| + u64 expected,
|
| + u64 *pCksum
|
| +){
|
| + int rc = p->rc;
|
| + if( pPrev->n==0 ){
|
| + fts5BufferSet(&rc, pPrev, n, (const u8*)z);
|
| + }else
|
| + if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){
|
| + u64 cksum3 = *pCksum;
|
| + const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */
|
| + int nTerm = pPrev->n-1; /* Size of zTerm in bytes */
|
| + int iIdx = (pPrev->p[0] - FTS5_MAIN_PREFIX);
|
| + int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX);
|
| + u64 ck1 = 0;
|
| + u64 ck2 = 0;
|
| +
|
| + /* Check that the results returned for ASC and DESC queries are
|
| + ** the same. If not, call this corruption. */
|
| + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1);
|
| + if( rc==SQLITE_OK ){
|
| + int f = flags|FTS5INDEX_QUERY_DESC;
|
| + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
|
| + }
|
| + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
|
| +
|
| + /* If this is a prefix query, check that the results returned if the
|
| + ** the index is disabled are the same. In both ASC and DESC order.
|
| + **
|
| + ** This check may only be performed if the hash table is empty. This
|
| + ** is because the hash table only supports a single scan query at
|
| + ** a time, and the multi-iter loop from which this function is called
|
| + ** is already performing such a scan. */
|
| + if( p->nPendingData==0 ){
|
| + if( iIdx>0 && rc==SQLITE_OK ){
|
| + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX;
|
| + ck2 = 0;
|
| + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
|
| + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
|
| + }
|
| + if( iIdx>0 && rc==SQLITE_OK ){
|
| + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC;
|
| + ck2 = 0;
|
| + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
|
| + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
|
| + }
|
| + }
|
| +
|
| + cksum3 ^= ck1;
|
| + fts5BufferSet(&rc, pPrev, n, (const u8*)z);
|
| +
|
| + if( rc==SQLITE_OK && cksum3!=expected ){
|
| + rc = FTS5_CORRUPT;
|
| + }
|
| + *pCksum = cksum3;
|
| + }
|
| + p->rc = rc;
|
| +}
|
| +
|
| +#else
|
| +# define fts5TestDlidxReverse(x,y,z)
|
| +# define fts5TestTerm(u,v,w,x,y,z)
|
| +#endif
|
| +
|
| +/*
|
| +** Check that:
|
| +**
|
| +** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and
|
| +** contain zero terms.
|
| +** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and
|
| +** contain zero rowids.
|
| +*/
|
| +static void fts5IndexIntegrityCheckEmpty(
|
| + Fts5Index *p,
|
| + Fts5StructureSegment *pSeg, /* Segment to check internal consistency */
|
| + int iFirst,
|
| + int iNoRowid,
|
| + int iLast
|
| +){
|
| + int i;
|
| +
|
| + /* Now check that the iter.nEmpty leaves following the current leaf
|
| + ** (a) exist and (b) contain no terms. */
|
| + for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){
|
| + Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i));
|
| + if( pLeaf ){
|
| + if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT;
|
| + if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT;
|
| + }
|
| + fts5DataRelease(pLeaf);
|
| + }
|
| +}
|
| +
|
| +static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){
|
| + int iTermOff = 0;
|
| + int ii;
|
| +
|
| + Fts5Buffer buf1 = {0,0,0};
|
| + Fts5Buffer buf2 = {0,0,0};
|
| +
|
| + ii = pLeaf->szLeaf;
|
| + while( ii<pLeaf->nn && p->rc==SQLITE_OK ){
|
| + int res;
|
| + int iOff;
|
| + int nIncr;
|
| +
|
| + ii += fts5GetVarint32(&pLeaf->p[ii], nIncr);
|
| + iTermOff += nIncr;
|
| + iOff = iTermOff;
|
| +
|
| + if( iOff>=pLeaf->szLeaf ){
|
| + p->rc = FTS5_CORRUPT;
|
| + }else if( iTermOff==nIncr ){
|
| + int nByte;
|
| + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte);
|
| + if( (iOff+nByte)>pLeaf->szLeaf ){
|
| + p->rc = FTS5_CORRUPT;
|
| + }else{
|
| + fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]);
|
| + }
|
| + }else{
|
| + int nKeep, nByte;
|
| + iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep);
|
| + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte);
|
| + if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){
|
| + p->rc = FTS5_CORRUPT;
|
| + }else{
|
| + buf1.n = nKeep;
|
| + fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]);
|
| + }
|
| +
|
| + if( p->rc==SQLITE_OK ){
|
| + res = fts5BufferCompare(&buf1, &buf2);
|
| + if( res<=0 ) p->rc = FTS5_CORRUPT;
|
| + }
|
| + }
|
| + fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p);
|
| + }
|
| +
|
| + fts5BufferFree(&buf1);
|
| + fts5BufferFree(&buf2);
|
| +}
|
| +
|
| +static void fts5IndexIntegrityCheckSegment(
|
| + Fts5Index *p, /* FTS5 backend object */
|
| + Fts5StructureSegment *pSeg /* Segment to check internal consistency */
|
| +){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + sqlite3_stmt *pStmt = 0;
|
| + int rc2;
|
| + int iIdxPrevLeaf = pSeg->pgnoFirst-1;
|
| + int iDlidxPrevLeaf = pSeg->pgnoLast;
|
| +
|
| + if( pSeg->pgnoFirst==0 ) return;
|
| +
|
| + fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf(
|
| + "SELECT segid, term, (pgno>>1), (pgno&1) FROM %Q.'%q_idx' WHERE segid=%d",
|
| + pConfig->zDb, pConfig->zName, pSeg->iSegid
|
| + ));
|
| +
|
| + /* Iterate through the b-tree hierarchy. */
|
| + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
|
| + i64 iRow; /* Rowid for this leaf */
|
| + Fts5Data *pLeaf; /* Data for this leaf */
|
| +
|
| + int nIdxTerm = sqlite3_column_bytes(pStmt, 1);
|
| + const char *zIdxTerm = (const char*)sqlite3_column_text(pStmt, 1);
|
| + int iIdxLeaf = sqlite3_column_int(pStmt, 2);
|
| + int bIdxDlidx = sqlite3_column_int(pStmt, 3);
|
| +
|
| + /* If the leaf in question has already been trimmed from the segment,
|
| + ** ignore this b-tree entry. Otherwise, load it into memory. */
|
| + if( iIdxLeaf<pSeg->pgnoFirst ) continue;
|
| + iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf);
|
| + pLeaf = fts5DataRead(p, iRow);
|
| + if( pLeaf==0 ) break;
|
| +
|
| + /* Check that the leaf contains at least one term, and that it is equal
|
| + ** to or larger than the split-key in zIdxTerm. Also check that if there
|
| + ** is also a rowid pointer within the leaf page header, it points to a
|
| + ** location before the term. */
|
| + if( pLeaf->nn<=pLeaf->szLeaf ){
|
| + p->rc = FTS5_CORRUPT;
|
| + }else{
|
| + int iOff; /* Offset of first term on leaf */
|
| + int iRowidOff; /* Offset of first rowid on leaf */
|
| + int nTerm; /* Size of term on leaf in bytes */
|
| + int res; /* Comparison of term and split-key */
|
| +
|
| + iOff = fts5LeafFirstTermOff(pLeaf);
|
| + iRowidOff = fts5LeafFirstRowidOff(pLeaf);
|
| + if( iRowidOff>=iOff ){
|
| + p->rc = FTS5_CORRUPT;
|
| + }else{
|
| + iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
|
| + res = memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm));
|
| + if( res==0 ) res = nTerm - nIdxTerm;
|
| + if( res<0 ) p->rc = FTS5_CORRUPT;
|
| + }
|
| +
|
| + fts5IntegrityCheckPgidx(p, pLeaf);
|
| + }
|
| + fts5DataRelease(pLeaf);
|
| + if( p->rc ) break;
|
| +
|
| + /* Now check that the iter.nEmpty leaves following the current leaf
|
| + ** (a) exist and (b) contain no terms. */
|
| + fts5IndexIntegrityCheckEmpty(
|
| + p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1
|
| + );
|
| + if( p->rc ) break;
|
| +
|
| + /* If there is a doclist-index, check that it looks right. */
|
| + if( bIdxDlidx ){
|
| + Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */
|
| + int iPrevLeaf = iIdxLeaf;
|
| + int iSegid = pSeg->iSegid;
|
| + int iPg = 0;
|
| + i64 iKey;
|
| +
|
| + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf);
|
| + fts5DlidxIterEof(p, pDlidx)==0;
|
| + fts5DlidxIterNext(p, pDlidx)
|
| + ){
|
| +
|
| + /* Check any rowid-less pages that occur before the current leaf. */
|
| + for(iPg=iPrevLeaf+1; iPg<fts5DlidxIterPgno(pDlidx); iPg++){
|
| + iKey = FTS5_SEGMENT_ROWID(iSegid, iPg);
|
| + pLeaf = fts5DataRead(p, iKey);
|
| + if( pLeaf ){
|
| + if( fts5LeafFirstRowidOff(pLeaf)!=0 ) p->rc = FTS5_CORRUPT;
|
| + fts5DataRelease(pLeaf);
|
| + }
|
| + }
|
| + iPrevLeaf = fts5DlidxIterPgno(pDlidx);
|
| +
|
| + /* Check that the leaf page indicated by the iterator really does
|
| + ** contain the rowid suggested by the same. */
|
| + iKey = FTS5_SEGMENT_ROWID(iSegid, iPrevLeaf);
|
| + pLeaf = fts5DataRead(p, iKey);
|
| + if( pLeaf ){
|
| + i64 iRowid;
|
| + int iRowidOff = fts5LeafFirstRowidOff(pLeaf);
|
| + ASSERT_SZLEAF_OK(pLeaf);
|
| + if( iRowidOff>=pLeaf->szLeaf ){
|
| + p->rc = FTS5_CORRUPT;
|
| + }else{
|
| + fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
|
| + if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT;
|
| + }
|
| + fts5DataRelease(pLeaf);
|
| + }
|
| + }
|
| +
|
| + iDlidxPrevLeaf = iPg;
|
| + fts5DlidxIterFree(pDlidx);
|
| + fts5TestDlidxReverse(p, iSegid, iIdxLeaf);
|
| + }else{
|
| + iDlidxPrevLeaf = pSeg->pgnoLast;
|
| + /* TODO: Check there is no doclist index */
|
| + }
|
| +
|
| + iIdxPrevLeaf = iIdxLeaf;
|
| + }
|
| +
|
| + rc2 = sqlite3_finalize(pStmt);
|
| + if( p->rc==SQLITE_OK ) p->rc = rc2;
|
| +
|
| + /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */
|
| +#if 0
|
| + if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){
|
| + p->rc = FTS5_CORRUPT;
|
| + }
|
| +#endif
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Run internal checks to ensure that the FTS index (a) is internally
|
| +** consistent and (b) contains entries for which the XOR of the checksums
|
| +** as calculated by fts5IndexEntryCksum() is cksum.
|
| +**
|
| +** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
|
| +** checksum does not match. Return SQLITE_OK if all checks pass without
|
| +** error, or some other SQLite error code if another error (e.g. OOM)
|
| +** occurs.
|
| +*/
|
| +static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
|
| + u64 cksum2 = 0; /* Checksum based on contents of indexes */
|
| + Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */
|
| + Fts5IndexIter *pIter; /* Used to iterate through entire index */
|
| + Fts5Structure *pStruct; /* Index structure */
|
| +
|
| +#ifdef SQLITE_DEBUG
|
| + /* Used by extra internal tests only run if NDEBUG is not defined */
|
| + u64 cksum3 = 0; /* Checksum based on contents of indexes */
|
| + Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */
|
| +#endif
|
| +
|
| + /* Load the FTS index structure */
|
| + pStruct = fts5StructureRead(p);
|
| +
|
| + /* Check that the internal nodes of each segment match the leaves */
|
| + if( pStruct ){
|
| + int iLvl, iSeg;
|
| + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
|
| + for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
|
| + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
|
| + fts5IndexIntegrityCheckSegment(p, pSeg);
|
| + }
|
| + }
|
| + }
|
| +
|
| + /* The cksum argument passed to this function is a checksum calculated
|
| + ** based on all expected entries in the FTS index (including prefix index
|
| + ** entries). This block checks that a checksum calculated based on the
|
| + ** actual contents of FTS index is identical.
|
| + **
|
| + ** Two versions of the same checksum are calculated. The first (stack
|
| + ** variable cksum2) based on entries extracted from the full-text index
|
| + ** while doing a linear scan of each individual index in turn.
|
| + **
|
| + ** As each term visited by the linear scans, a separate query for the
|
| + ** same term is performed. cksum3 is calculated based on the entries
|
| + ** extracted by these queries.
|
| + */
|
| + for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, -1, 0, &pIter);
|
| + fts5MultiIterEof(p, pIter)==0;
|
| + fts5MultiIterNext(p, pIter, 0, 0)
|
| + ){
|
| + int n; /* Size of term in bytes */
|
| + i64 iPos = 0; /* Position read from poslist */
|
| + int iOff = 0; /* Offset within poslist */
|
| + i64 iRowid = fts5MultiIterRowid(pIter);
|
| + char *z = (char*)fts5MultiIterTerm(pIter, &n);
|
| +
|
| + /* If this is a new term, query for it. Update cksum3 with the results. */
|
| + fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
|
| +
|
| + poslist.n = 0;
|
| + fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist);
|
| + while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
|
| + int iCol = FTS5_POS2COLUMN(iPos);
|
| + int iTokOff = FTS5_POS2OFFSET(iPos);
|
| + cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
|
| + }
|
| + }
|
| + fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
|
| +
|
| + fts5MultiIterFree(p, pIter);
|
| + if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
|
| +
|
| + fts5StructureRelease(pStruct);
|
| +#ifdef SQLITE_DEBUG
|
| + fts5BufferFree(&term);
|
| +#endif
|
| + fts5BufferFree(&poslist);
|
| + return fts5IndexReturn(p);
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Calculate and return a checksum that is the XOR of the index entry
|
| +** checksum of all entries that would be generated by the token specified
|
| +** by the final 5 arguments.
|
| +*/
|
| +static u64 sqlite3Fts5IndexCksum(
|
| + Fts5Config *pConfig, /* Configuration object */
|
| + i64 iRowid, /* Document term appears in */
|
| + int iCol, /* Column term appears in */
|
| + int iPos, /* Position term appears in */
|
| + const char *pTerm, int nTerm /* Term at iPos */
|
| +){
|
| + u64 ret = 0; /* Return value */
|
| + int iIdx; /* For iterating through indexes */
|
| +
|
| + ret = fts5IndexEntryCksum(iRowid, iCol, iPos, 0, pTerm, nTerm);
|
| +
|
| + for(iIdx=0; iIdx<pConfig->nPrefix; iIdx++){
|
| + int nByte = fts5IndexCharlenToBytelen(pTerm, nTerm, pConfig->aPrefix[iIdx]);
|
| + if( nByte ){
|
| + ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, iIdx+1, pTerm, nByte);
|
| + }
|
| + }
|
| +
|
| + return ret;
|
| +}
|
| +
|
| +/*************************************************************************
|
| +**************************************************************************
|
| +** Below this point is the implementation of the fts5_decode() scalar
|
| +** function only.
|
| +*/
|
| +
|
| +/*
|
| +** Decode a segment-data rowid from the %_data table. This function is
|
| +** the opposite of macro FTS5_SEGMENT_ROWID().
|
| +*/
|
| +static void fts5DecodeRowid(
|
| + i64 iRowid, /* Rowid from %_data table */
|
| + int *piSegid, /* OUT: Segment id */
|
| + int *pbDlidx, /* OUT: Dlidx flag */
|
| + int *piHeight, /* OUT: Height */
|
| + int *piPgno /* OUT: Page number */
|
| +){
|
| + *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1));
|
| + iRowid >>= FTS5_DATA_PAGE_B;
|
| +
|
| + *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1));
|
| + iRowid >>= FTS5_DATA_HEIGHT_B;
|
| +
|
| + *pbDlidx = (int)(iRowid & 0x0001);
|
| + iRowid >>= FTS5_DATA_DLI_B;
|
| +
|
| + *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
|
| +}
|
| +
|
| +static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
|
| + int iSegid, iHeight, iPgno, bDlidx; /* Rowid compenents */
|
| + fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno);
|
| +
|
| + if( iSegid==0 ){
|
| + if( iKey==FTS5_AVERAGES_ROWID ){
|
| + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{averages} ");
|
| + }else{
|
| + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{structure}");
|
| + }
|
| + }
|
| + else{
|
| + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}",
|
| + bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno
|
| + );
|
| + }
|
| +}
|
| +
|
| +static void fts5DebugStructure(
|
| + int *pRc, /* IN/OUT: error code */
|
| + Fts5Buffer *pBuf,
|
| + Fts5Structure *p
|
| +){
|
| + int iLvl, iSeg; /* Iterate through levels, segments */
|
| +
|
| + for(iLvl=0; iLvl<p->nLevel; iLvl++){
|
| + Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
|
| + sqlite3Fts5BufferAppendPrintf(pRc, pBuf,
|
| + " {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg
|
| + );
|
| + for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
|
| + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
|
| + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}",
|
| + pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast
|
| + );
|
| + }
|
| + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** This is part of the fts5_decode() debugging aid.
|
| +**
|
| +** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This
|
| +** function appends a human-readable representation of the same object
|
| +** to the buffer passed as the second argument.
|
| +*/
|
| +static void fts5DecodeStructure(
|
| + int *pRc, /* IN/OUT: error code */
|
| + Fts5Buffer *pBuf,
|
| + const u8 *pBlob, int nBlob
|
| +){
|
| + int rc; /* Return code */
|
| + Fts5Structure *p = 0; /* Decoded structure object */
|
| +
|
| + rc = fts5StructureDecode(pBlob, nBlob, 0, &p);
|
| + if( rc!=SQLITE_OK ){
|
| + *pRc = rc;
|
| + return;
|
| + }
|
| +
|
| + fts5DebugStructure(pRc, pBuf, p);
|
| + fts5StructureRelease(p);
|
| +}
|
| +
|
| +/*
|
| +** This is part of the fts5_decode() debugging aid.
|
| +**
|
| +** Arguments pBlob/nBlob contain an "averages" record. This function
|
| +** appends a human-readable representation of record to the buffer passed
|
| +** as the second argument.
|
| +*/
|
| +static void fts5DecodeAverages(
|
| + int *pRc, /* IN/OUT: error code */
|
| + Fts5Buffer *pBuf,
|
| + const u8 *pBlob, int nBlob
|
| +){
|
| + int i = 0;
|
| + const char *zSpace = "";
|
| +
|
| + while( i<nBlob ){
|
| + u64 iVal;
|
| + i += sqlite3Fts5GetVarint(&pBlob[i], &iVal);
|
| + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "%s%d", zSpace, (int)iVal);
|
| + zSpace = " ";
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Buffer (a/n) is assumed to contain a list of serialized varints. Read
|
| +** each varint and append its string representation to buffer pBuf. Return
|
| +** after either the input buffer is exhausted or a 0 value is read.
|
| +**
|
| +** The return value is the number of bytes read from the input buffer.
|
| +*/
|
| +static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
|
| + int iOff = 0;
|
| + while( iOff<n ){
|
| + int iVal;
|
| + iOff += fts5GetVarint32(&a[iOff], iVal);
|
| + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal);
|
| + }
|
| + return iOff;
|
| +}
|
| +
|
| +/*
|
| +** The start of buffer (a/n) contains the start of a doclist. The doclist
|
| +** may or may not finish within the buffer. This function appends a text
|
| +** representation of the part of the doclist that is present to buffer
|
| +** pBuf.
|
| +**
|
| +** The return value is the number of bytes read from the input buffer.
|
| +*/
|
| +static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
|
| + i64 iDocid = 0;
|
| + int iOff = 0;
|
| +
|
| + if( n>0 ){
|
| + iOff = sqlite3Fts5GetVarint(a, (u64*)&iDocid);
|
| + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
|
| + }
|
| + while( iOff<n ){
|
| + int nPos;
|
| + int bDel;
|
| + iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDel);
|
| + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " nPos=%d%s", nPos, bDel?"*":"");
|
| + iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
|
| + if( iOff<n ){
|
| + i64 iDelta;
|
| + iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDelta);
|
| + iDocid += iDelta;
|
| + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
|
| + }
|
| + }
|
| +
|
| + return iOff;
|
| +}
|
| +
|
| +/*
|
| +** The implementation of user-defined scalar function fts5_decode().
|
| +*/
|
| +static void fts5DecodeFunction(
|
| + sqlite3_context *pCtx, /* Function call context */
|
| + int nArg, /* Number of args (always 2) */
|
| + sqlite3_value **apVal /* Function arguments */
|
| +){
|
| + i64 iRowid; /* Rowid for record being decoded */
|
| + int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */
|
| + const u8 *aBlob; int n; /* Record to decode */
|
| + u8 *a = 0;
|
| + Fts5Buffer s; /* Build up text to return here */
|
| + int rc = SQLITE_OK; /* Return code */
|
| + int nSpace = 0;
|
| +
|
| + assert( nArg==2 );
|
| + memset(&s, 0, sizeof(Fts5Buffer));
|
| + iRowid = sqlite3_value_int64(apVal[0]);
|
| +
|
| + /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[]
|
| + ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents
|
| + ** buffer overreads even if the record is corrupt. */
|
| + n = sqlite3_value_bytes(apVal[1]);
|
| + aBlob = sqlite3_value_blob(apVal[1]);
|
| + nSpace = n + FTS5_DATA_ZERO_PADDING;
|
| + a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace);
|
| + if( a==0 ) goto decode_out;
|
| + memcpy(a, aBlob, n);
|
| +
|
| +
|
| + fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno);
|
| +
|
| + fts5DebugRowid(&rc, &s, iRowid);
|
| + if( bDlidx ){
|
| + Fts5Data dlidx;
|
| + Fts5DlidxLvl lvl;
|
| +
|
| + dlidx.p = a;
|
| + dlidx.nn = n;
|
| +
|
| + memset(&lvl, 0, sizeof(Fts5DlidxLvl));
|
| + lvl.pData = &dlidx;
|
| + lvl.iLeafPgno = iPgno;
|
| +
|
| + for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){
|
| + sqlite3Fts5BufferAppendPrintf(&rc, &s,
|
| + " %d(%lld)", lvl.iLeafPgno, lvl.iRowid
|
| + );
|
| + }
|
| + }else if( iSegid==0 ){
|
| + if( iRowid==FTS5_AVERAGES_ROWID ){
|
| + fts5DecodeAverages(&rc, &s, a, n);
|
| + }else{
|
| + fts5DecodeStructure(&rc, &s, a, n);
|
| + }
|
| + }else{
|
| + Fts5Buffer term; /* Current term read from page */
|
| + int szLeaf; /* Offset of pgidx in a[] */
|
| + int iPgidxOff;
|
| + int iPgidxPrev = 0; /* Previous value read from pgidx */
|
| + int iTermOff = 0;
|
| + int iRowidOff = 0;
|
| + int iOff;
|
| + int nDoclist;
|
| +
|
| + memset(&term, 0, sizeof(Fts5Buffer));
|
| +
|
| + if( n<4 ){
|
| + sqlite3Fts5BufferSet(&rc, &s, 7, (const u8*)"corrupt");
|
| + goto decode_out;
|
| + }else{
|
| + iRowidOff = fts5GetU16(&a[0]);
|
| + iPgidxOff = szLeaf = fts5GetU16(&a[2]);
|
| + if( iPgidxOff<n ){
|
| + fts5GetVarint32(&a[iPgidxOff], iTermOff);
|
| + }
|
| + }
|
| +
|
| + /* Decode the position list tail at the start of the page */
|
| + if( iRowidOff!=0 ){
|
| + iOff = iRowidOff;
|
| + }else if( iTermOff!=0 ){
|
| + iOff = iTermOff;
|
| + }else{
|
| + iOff = szLeaf;
|
| + }
|
| + fts5DecodePoslist(&rc, &s, &a[4], iOff-4);
|
| +
|
| + /* Decode any more doclist data that appears on the page before the
|
| + ** first term. */
|
| + nDoclist = (iTermOff ? iTermOff : szLeaf) - iOff;
|
| + fts5DecodeDoclist(&rc, &s, &a[iOff], nDoclist);
|
| +
|
| + while( iPgidxOff<n ){
|
| + int bFirst = (iPgidxOff==szLeaf); /* True for first term on page */
|
| + int nByte; /* Bytes of data */
|
| + int iEnd;
|
| +
|
| + iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nByte);
|
| + iPgidxPrev += nByte;
|
| + iOff = iPgidxPrev;
|
| +
|
| + if( iPgidxOff<n ){
|
| + fts5GetVarint32(&a[iPgidxOff], nByte);
|
| + iEnd = iPgidxPrev + nByte;
|
| + }else{
|
| + iEnd = szLeaf;
|
| + }
|
| +
|
| + if( bFirst==0 ){
|
| + iOff += fts5GetVarint32(&a[iOff], nByte);
|
| + term.n = nByte;
|
| + }
|
| + iOff += fts5GetVarint32(&a[iOff], nByte);
|
| + fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]);
|
| + iOff += nByte;
|
| +
|
| + sqlite3Fts5BufferAppendPrintf(
|
| + &rc, &s, " term=%.*s", term.n, (const char*)term.p
|
| + );
|
| + iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], iEnd-iOff);
|
| + }
|
| +
|
| + fts5BufferFree(&term);
|
| + }
|
| +
|
| + decode_out:
|
| + sqlite3_free(a);
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT);
|
| + }else{
|
| + sqlite3_result_error_code(pCtx, rc);
|
| + }
|
| + fts5BufferFree(&s);
|
| +}
|
| +
|
| +/*
|
| +** The implementation of user-defined scalar function fts5_rowid().
|
| +*/
|
| +static void fts5RowidFunction(
|
| + sqlite3_context *pCtx, /* Function call context */
|
| + int nArg, /* Number of args (always 2) */
|
| + sqlite3_value **apVal /* Function arguments */
|
| +){
|
| + const char *zArg;
|
| + if( nArg==0 ){
|
| + sqlite3_result_error(pCtx, "should be: fts5_rowid(subject, ....)", -1);
|
| + }else{
|
| + zArg = (const char*)sqlite3_value_text(apVal[0]);
|
| + if( 0==sqlite3_stricmp(zArg, "segment") ){
|
| + i64 iRowid;
|
| + int segid, pgno;
|
| + if( nArg!=3 ){
|
| + sqlite3_result_error(pCtx,
|
| + "should be: fts5_rowid('segment', segid, pgno))", -1
|
| + );
|
| + }else{
|
| + segid = sqlite3_value_int(apVal[1]);
|
| + pgno = sqlite3_value_int(apVal[2]);
|
| + iRowid = FTS5_SEGMENT_ROWID(segid, pgno);
|
| + sqlite3_result_int64(pCtx, iRowid);
|
| + }
|
| + }else{
|
| + sqlite3_result_error(pCtx,
|
| + "first arg to fts5_rowid() must be 'segment'" , -1
|
| + );
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** This is called as part of registering the FTS5 module with database
|
| +** connection db. It registers several user-defined scalar functions useful
|
| +** with FTS5.
|
| +**
|
| +** If successful, SQLITE_OK is returned. If an error occurs, some other
|
| +** SQLite error code is returned instead.
|
| +*/
|
| +static int sqlite3Fts5IndexInit(sqlite3 *db){
|
| + int rc = sqlite3_create_function(
|
| + db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
|
| + );
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3_create_function(
|
| + db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
|
| + );
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** 2014 Jun 09
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +** This is an SQLite module implementing full-text search.
|
| +*/
|
| +
|
| +
|
| +/* #include "fts5Int.h" */
|
| +
|
| +/*
|
| +** This variable is set to false when running tests for which the on disk
|
| +** structures should not be corrupt. Otherwise, true. If it is false, extra
|
| +** assert() conditions in the fts5 code are activated - conditions that are
|
| +** only true if it is guaranteed that the fts5 database is not corrupt.
|
| +*/
|
| +SQLITE_API int sqlite3_fts5_may_be_corrupt = 1;
|
| +
|
| +
|
| +typedef struct Fts5Auxdata Fts5Auxdata;
|
| +typedef struct Fts5Auxiliary Fts5Auxiliary;
|
| +typedef struct Fts5Cursor Fts5Cursor;
|
| +typedef struct Fts5Sorter Fts5Sorter;
|
| +typedef struct Fts5Table Fts5Table;
|
| +typedef struct Fts5TokenizerModule Fts5TokenizerModule;
|
| +
|
| +/*
|
| +** NOTES ON TRANSACTIONS:
|
| +**
|
| +** SQLite invokes the following virtual table methods as transactions are
|
| +** opened and closed by the user:
|
| +**
|
| +** xBegin(): Start of a new transaction.
|
| +** xSync(): Initial part of two-phase commit.
|
| +** xCommit(): Final part of two-phase commit.
|
| +** xRollback(): Rollback the transaction.
|
| +**
|
| +** Anything that is required as part of a commit that may fail is performed
|
| +** in the xSync() callback. Current versions of SQLite ignore any errors
|
| +** returned by xCommit().
|
| +**
|
| +** And as sub-transactions are opened/closed:
|
| +**
|
| +** xSavepoint(int S): Open savepoint S.
|
| +** xRelease(int S): Commit and close savepoint S.
|
| +** xRollbackTo(int S): Rollback to start of savepoint S.
|
| +**
|
| +** During a write-transaction the fts5_index.c module may cache some data
|
| +** in-memory. It is flushed to disk whenever xSync(), xRelease() or
|
| +** xSavepoint() is called. And discarded whenever xRollback() or xRollbackTo()
|
| +** is called.
|
| +**
|
| +** Additionally, if SQLITE_DEBUG is defined, an instance of the following
|
| +** structure is used to record the current transaction state. This information
|
| +** is not required, but it is used in the assert() statements executed by
|
| +** function fts5CheckTransactionState() (see below).
|
| +*/
|
| +struct Fts5TransactionState {
|
| + int eState; /* 0==closed, 1==open, 2==synced */
|
| + int iSavepoint; /* Number of open savepoints (0 -> none) */
|
| +};
|
| +
|
| +/*
|
| +** A single object of this type is allocated when the FTS5 module is
|
| +** registered with a database handle. It is used to store pointers to
|
| +** all registered FTS5 extensions - tokenizers and auxiliary functions.
|
| +*/
|
| +struct Fts5Global {
|
| + fts5_api api; /* User visible part of object (see fts5.h) */
|
| + sqlite3 *db; /* Associated database connection */
|
| + i64 iNextId; /* Used to allocate unique cursor ids */
|
| + Fts5Auxiliary *pAux; /* First in list of all aux. functions */
|
| + Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */
|
| + Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */
|
| + Fts5Cursor *pCsr; /* First in list of all open cursors */
|
| +};
|
| +
|
| +/*
|
| +** Each auxiliary function registered with the FTS5 module is represented
|
| +** by an object of the following type. All such objects are stored as part
|
| +** of the Fts5Global.pAux list.
|
| +*/
|
| +struct Fts5Auxiliary {
|
| + Fts5Global *pGlobal; /* Global context for this function */
|
| + char *zFunc; /* Function name (nul-terminated) */
|
| + void *pUserData; /* User-data pointer */
|
| + fts5_extension_function xFunc; /* Callback function */
|
| + void (*xDestroy)(void*); /* Destructor function */
|
| + Fts5Auxiliary *pNext; /* Next registered auxiliary function */
|
| +};
|
| +
|
| +/*
|
| +** Each tokenizer module registered with the FTS5 module is represented
|
| +** by an object of the following type. All such objects are stored as part
|
| +** of the Fts5Global.pTok list.
|
| +*/
|
| +struct Fts5TokenizerModule {
|
| + char *zName; /* Name of tokenizer */
|
| + void *pUserData; /* User pointer passed to xCreate() */
|
| + fts5_tokenizer x; /* Tokenizer functions */
|
| + void (*xDestroy)(void*); /* Destructor function */
|
| + Fts5TokenizerModule *pNext; /* Next registered tokenizer module */
|
| +};
|
| +
|
| +/*
|
| +** Virtual-table object.
|
| +*/
|
| +struct Fts5Table {
|
| + sqlite3_vtab base; /* Base class used by SQLite core */
|
| + Fts5Config *pConfig; /* Virtual table configuration */
|
| + Fts5Index *pIndex; /* Full-text index */
|
| + Fts5Storage *pStorage; /* Document store */
|
| + Fts5Global *pGlobal; /* Global (connection wide) data */
|
| + Fts5Cursor *pSortCsr; /* Sort data from this cursor */
|
| +#ifdef SQLITE_DEBUG
|
| + struct Fts5TransactionState ts;
|
| +#endif
|
| +};
|
| +
|
| +struct Fts5MatchPhrase {
|
| + Fts5Buffer *pPoslist; /* Pointer to current poslist */
|
| + int nTerm; /* Size of phrase in terms */
|
| +};
|
| +
|
| +/*
|
| +** pStmt:
|
| +** SELECT rowid, <fts> FROM <fts> ORDER BY +rank;
|
| +**
|
| +** aIdx[]:
|
| +** There is one entry in the aIdx[] array for each phrase in the query,
|
| +** the value of which is the offset within aPoslist[] following the last
|
| +** byte of the position list for the corresponding phrase.
|
| +*/
|
| +struct Fts5Sorter {
|
| + sqlite3_stmt *pStmt;
|
| + i64 iRowid; /* Current rowid */
|
| + const u8 *aPoslist; /* Position lists for current row */
|
| + int nIdx; /* Number of entries in aIdx[] */
|
| + int aIdx[1]; /* Offsets into aPoslist for current row */
|
| +};
|
| +
|
| +
|
| +/*
|
| +** Virtual-table cursor object.
|
| +**
|
| +** iSpecial:
|
| +** If this is a 'special' query (refer to function fts5SpecialMatch()),
|
| +** then this variable contains the result of the query.
|
| +**
|
| +** iFirstRowid, iLastRowid:
|
| +** These variables are only used for FTS5_PLAN_MATCH cursors. Assuming the
|
| +** cursor iterates in ascending order of rowids, iFirstRowid is the lower
|
| +** limit of rowids to return, and iLastRowid the upper. In other words, the
|
| +** WHERE clause in the user's query might have been:
|
| +**
|
| +** <tbl> MATCH <expr> AND rowid BETWEEN $iFirstRowid AND $iLastRowid
|
| +**
|
| +** If the cursor iterates in descending order of rowid, iFirstRowid
|
| +** is the upper limit (i.e. the "first" rowid visited) and iLastRowid
|
| +** the lower.
|
| +*/
|
| +struct Fts5Cursor {
|
| + sqlite3_vtab_cursor base; /* Base class used by SQLite core */
|
| + Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */
|
| + int *aColumnSize; /* Values for xColumnSize() */
|
| + i64 iCsrId; /* Cursor id */
|
| +
|
| + /* Zero from this point onwards on cursor reset */
|
| + int ePlan; /* FTS5_PLAN_XXX value */
|
| + int bDesc; /* True for "ORDER BY rowid DESC" queries */
|
| + i64 iFirstRowid; /* Return no rowids earlier than this */
|
| + i64 iLastRowid; /* Return no rowids later than this */
|
| + sqlite3_stmt *pStmt; /* Statement used to read %_content */
|
| + Fts5Expr *pExpr; /* Expression for MATCH queries */
|
| + Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */
|
| + int csrflags; /* Mask of cursor flags (see below) */
|
| + i64 iSpecial; /* Result of special query */
|
| +
|
| + /* "rank" function. Populated on demand from vtab.xColumn(). */
|
| + char *zRank; /* Custom rank function */
|
| + char *zRankArgs; /* Custom rank function args */
|
| + Fts5Auxiliary *pRank; /* Rank callback (or NULL) */
|
| + int nRankArg; /* Number of trailing arguments for rank() */
|
| + sqlite3_value **apRankArg; /* Array of trailing arguments */
|
| + sqlite3_stmt *pRankArgStmt; /* Origin of objects in apRankArg[] */
|
| +
|
| + /* Auxiliary data storage */
|
| + Fts5Auxiliary *pAux; /* Currently executing extension function */
|
| + Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */
|
| +
|
| + /* Cache used by auxiliary functions xInst() and xInstCount() */
|
| + Fts5PoslistReader *aInstIter; /* One for each phrase */
|
| + int nInstAlloc; /* Size of aInst[] array (entries / 3) */
|
| + int nInstCount; /* Number of phrase instances */
|
| + int *aInst; /* 3 integers per phrase instance */
|
| +};
|
| +
|
| +/*
|
| +** Bits that make up the "idxNum" parameter passed indirectly by
|
| +** xBestIndex() to xFilter().
|
| +*/
|
| +#define FTS5_BI_MATCH 0x0001 /* <tbl> MATCH ? */
|
| +#define FTS5_BI_RANK 0x0002 /* rank MATCH ? */
|
| +#define FTS5_BI_ROWID_EQ 0x0004 /* rowid == ? */
|
| +#define FTS5_BI_ROWID_LE 0x0008 /* rowid <= ? */
|
| +#define FTS5_BI_ROWID_GE 0x0010 /* rowid >= ? */
|
| +
|
| +#define FTS5_BI_ORDER_RANK 0x0020
|
| +#define FTS5_BI_ORDER_ROWID 0x0040
|
| +#define FTS5_BI_ORDER_DESC 0x0080
|
| +
|
| +/*
|
| +** Values for Fts5Cursor.csrflags
|
| +*/
|
| +#define FTS5CSR_REQUIRE_CONTENT 0x01
|
| +#define FTS5CSR_REQUIRE_DOCSIZE 0x02
|
| +#define FTS5CSR_REQUIRE_INST 0x04
|
| +#define FTS5CSR_EOF 0x08
|
| +#define FTS5CSR_FREE_ZRANK 0x10
|
| +#define FTS5CSR_REQUIRE_RESEEK 0x20
|
| +
|
| +#define BitFlagAllTest(x,y) (((x) & (y))==(y))
|
| +#define BitFlagTest(x,y) (((x) & (y))!=0)
|
| +
|
| +
|
| +/*
|
| +** Macros to Set(), Clear() and Test() cursor flags.
|
| +*/
|
| +#define CsrFlagSet(pCsr, flag) ((pCsr)->csrflags |= (flag))
|
| +#define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag))
|
| +#define CsrFlagTest(pCsr, flag) ((pCsr)->csrflags & (flag))
|
| +
|
| +struct Fts5Auxdata {
|
| + Fts5Auxiliary *pAux; /* Extension to which this belongs */
|
| + void *pPtr; /* Pointer value */
|
| + void(*xDelete)(void*); /* Destructor */
|
| + Fts5Auxdata *pNext; /* Next object in linked list */
|
| +};
|
| +
|
| +#ifdef SQLITE_DEBUG
|
| +#define FTS5_BEGIN 1
|
| +#define FTS5_SYNC 2
|
| +#define FTS5_COMMIT 3
|
| +#define FTS5_ROLLBACK 4
|
| +#define FTS5_SAVEPOINT 5
|
| +#define FTS5_RELEASE 6
|
| +#define FTS5_ROLLBACKTO 7
|
| +static void fts5CheckTransactionState(Fts5Table *p, int op, int iSavepoint){
|
| + switch( op ){
|
| + case FTS5_BEGIN:
|
| + assert( p->ts.eState==0 );
|
| + p->ts.eState = 1;
|
| + p->ts.iSavepoint = -1;
|
| + break;
|
| +
|
| + case FTS5_SYNC:
|
| + assert( p->ts.eState==1 );
|
| + p->ts.eState = 2;
|
| + break;
|
| +
|
| + case FTS5_COMMIT:
|
| + assert( p->ts.eState==2 );
|
| + p->ts.eState = 0;
|
| + break;
|
| +
|
| + case FTS5_ROLLBACK:
|
| + assert( p->ts.eState==1 || p->ts.eState==2 || p->ts.eState==0 );
|
| + p->ts.eState = 0;
|
| + break;
|
| +
|
| + case FTS5_SAVEPOINT:
|
| + assert( p->ts.eState==1 );
|
| + assert( iSavepoint>=0 );
|
| + assert( iSavepoint>p->ts.iSavepoint );
|
| + p->ts.iSavepoint = iSavepoint;
|
| + break;
|
| +
|
| + case FTS5_RELEASE:
|
| + assert( p->ts.eState==1 );
|
| + assert( iSavepoint>=0 );
|
| + assert( iSavepoint<=p->ts.iSavepoint );
|
| + p->ts.iSavepoint = iSavepoint-1;
|
| + break;
|
| +
|
| + case FTS5_ROLLBACKTO:
|
| + assert( p->ts.eState==1 );
|
| + assert( iSavepoint>=0 );
|
| + assert( iSavepoint<=p->ts.iSavepoint );
|
| + p->ts.iSavepoint = iSavepoint;
|
| + break;
|
| + }
|
| +}
|
| +#else
|
| +# define fts5CheckTransactionState(x,y,z)
|
| +#endif
|
| +
|
| +/*
|
| +** Return true if pTab is a contentless table.
|
| +*/
|
| +static int fts5IsContentless(Fts5Table *pTab){
|
| + return pTab->pConfig->eContent==FTS5_CONTENT_NONE;
|
| +}
|
| +
|
| +/*
|
| +** Delete a virtual table handle allocated by fts5InitVtab().
|
| +*/
|
| +static void fts5FreeVtab(Fts5Table *pTab){
|
| + if( pTab ){
|
| + sqlite3Fts5IndexClose(pTab->pIndex);
|
| + sqlite3Fts5StorageClose(pTab->pStorage);
|
| + sqlite3Fts5ConfigFree(pTab->pConfig);
|
| + sqlite3_free(pTab);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** The xDisconnect() virtual table method.
|
| +*/
|
| +static int fts5DisconnectMethod(sqlite3_vtab *pVtab){
|
| + fts5FreeVtab((Fts5Table*)pVtab);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** The xDestroy() virtual table method.
|
| +*/
|
| +static int fts5DestroyMethod(sqlite3_vtab *pVtab){
|
| + Fts5Table *pTab = (Fts5Table*)pVtab;
|
| + int rc = sqlite3Fts5DropAll(pTab->pConfig);
|
| + if( rc==SQLITE_OK ){
|
| + fts5FreeVtab((Fts5Table*)pVtab);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** This function is the implementation of both the xConnect and xCreate
|
| +** methods of the FTS3 virtual table.
|
| +**
|
| +** The argv[] array contains the following:
|
| +**
|
| +** argv[0] -> module name ("fts5")
|
| +** argv[1] -> database name
|
| +** argv[2] -> table name
|
| +** argv[...] -> "column name" and other module argument fields.
|
| +*/
|
| +static int fts5InitVtab(
|
| + int bCreate, /* True for xCreate, false for xConnect */
|
| + sqlite3 *db, /* The SQLite database connection */
|
| + void *pAux, /* Hash table containing tokenizers */
|
| + int argc, /* Number of elements in argv array */
|
| + const char * const *argv, /* xCreate/xConnect argument array */
|
| + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
|
| + char **pzErr /* Write any error message here */
|
| +){
|
| + Fts5Global *pGlobal = (Fts5Global*)pAux;
|
| + const char **azConfig = (const char**)argv;
|
| + int rc = SQLITE_OK; /* Return code */
|
| + Fts5Config *pConfig = 0; /* Results of parsing argc/argv */
|
| + Fts5Table *pTab = 0; /* New virtual table object */
|
| +
|
| + /* Allocate the new vtab object and parse the configuration */
|
| + pTab = (Fts5Table*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Table));
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
|
| + assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + pTab->pConfig = pConfig;
|
| + pTab->pGlobal = pGlobal;
|
| + }
|
| +
|
| + /* Open the index sub-system */
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->pIndex, pzErr);
|
| + }
|
| +
|
| + /* Open the storage sub-system */
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5StorageOpen(
|
| + pConfig, pTab->pIndex, bCreate, &pTab->pStorage, pzErr
|
| + );
|
| + }
|
| +
|
| + /* Call sqlite3_declare_vtab() */
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
|
| + }
|
| +
|
| + /* Load the initial configuration */
|
| + if( rc==SQLITE_OK ){
|
| + assert( pConfig->pzErrmsg==0 );
|
| + pConfig->pzErrmsg = pzErr;
|
| + rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex);
|
| + sqlite3Fts5IndexRollback(pTab->pIndex);
|
| + pConfig->pzErrmsg = 0;
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + fts5FreeVtab(pTab);
|
| + pTab = 0;
|
| + }else if( bCreate ){
|
| + fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
|
| + }
|
| + *ppVTab = (sqlite3_vtab*)pTab;
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** The xConnect() and xCreate() methods for the virtual table. All the
|
| +** work is done in function fts5InitVtab().
|
| +*/
|
| +static int fts5ConnectMethod(
|
| + sqlite3 *db, /* Database connection */
|
| + void *pAux, /* Pointer to tokenizer hash table */
|
| + int argc, /* Number of elements in argv array */
|
| + const char * const *argv, /* xCreate/xConnect argument array */
|
| + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
|
| + char **pzErr /* OUT: sqlite3_malloc'd error message */
|
| +){
|
| + return fts5InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
|
| +}
|
| +static int fts5CreateMethod(
|
| + sqlite3 *db, /* Database connection */
|
| + void *pAux, /* Pointer to tokenizer hash table */
|
| + int argc, /* Number of elements in argv array */
|
| + const char * const *argv, /* xCreate/xConnect argument array */
|
| + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
|
| + char **pzErr /* OUT: sqlite3_malloc'd error message */
|
| +){
|
| + return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
|
| +}
|
| +
|
| +/*
|
| +** The different query plans.
|
| +*/
|
| +#define FTS5_PLAN_MATCH 1 /* (<tbl> MATCH ?) */
|
| +#define FTS5_PLAN_SOURCE 2 /* A source cursor for SORTED_MATCH */
|
| +#define FTS5_PLAN_SPECIAL 3 /* An internal query */
|
| +#define FTS5_PLAN_SORTED_MATCH 4 /* (<tbl> MATCH ? ORDER BY rank) */
|
| +#define FTS5_PLAN_SCAN 5 /* No usable constraint */
|
| +#define FTS5_PLAN_ROWID 6 /* (rowid = ?) */
|
| +
|
| +/*
|
| +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this
|
| +** extension is currently being used by a version of SQLite too old to
|
| +** support index-info flags. In that case this function is a no-op.
|
| +*/
|
| +static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){
|
| +#if SQLITE_VERSION_NUMBER>=3008012
|
| +#ifndef SQLITE_CORE
|
| + if( sqlite3_libversion_number()>=3008012 )
|
| +#endif
|
| + {
|
| + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE;
|
| + }
|
| +#endif
|
| +}
|
| +
|
| +/*
|
| +** Implementation of the xBestIndex method for FTS5 tables. Within the
|
| +** WHERE constraint, it searches for the following:
|
| +**
|
| +** 1. A MATCH constraint against the special column.
|
| +** 2. A MATCH constraint against the "rank" column.
|
| +** 3. An == constraint against the rowid column.
|
| +** 4. A < or <= constraint against the rowid column.
|
| +** 5. A > or >= constraint against the rowid column.
|
| +**
|
| +** Within the ORDER BY, either:
|
| +**
|
| +** 5. ORDER BY rank [ASC|DESC]
|
| +** 6. ORDER BY rowid [ASC|DESC]
|
| +**
|
| +** Costs are assigned as follows:
|
| +**
|
| +** a) If an unusable MATCH operator is present in the WHERE clause, the
|
| +** cost is unconditionally set to 1e50 (a really big number).
|
| +**
|
| +** a) If a MATCH operator is present, the cost depends on the other
|
| +** constraints also present. As follows:
|
| +**
|
| +** * No other constraints: cost=1000.0
|
| +** * One rowid range constraint: cost=750.0
|
| +** * Both rowid range constraints: cost=500.0
|
| +** * An == rowid constraint: cost=100.0
|
| +**
|
| +** b) Otherwise, if there is no MATCH:
|
| +**
|
| +** * No other constraints: cost=1000000.0
|
| +** * One rowid range constraint: cost=750000.0
|
| +** * Both rowid range constraints: cost=250000.0
|
| +** * An == rowid constraint: cost=10.0
|
| +**
|
| +** Costs are not modified by the ORDER BY clause.
|
| +*/
|
| +static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
|
| + Fts5Table *pTab = (Fts5Table*)pVTab;
|
| + Fts5Config *pConfig = pTab->pConfig;
|
| + int idxFlags = 0; /* Parameter passed through to xFilter() */
|
| + int bHasMatch;
|
| + int iNext;
|
| + int i;
|
| +
|
| + struct Constraint {
|
| + int op; /* Mask against sqlite3_index_constraint.op */
|
| + int fts5op; /* FTS5 mask for idxFlags */
|
| + int iCol; /* 0==rowid, 1==tbl, 2==rank */
|
| + int omit; /* True to omit this if found */
|
| + int iConsIndex; /* Index in pInfo->aConstraint[] */
|
| + } aConstraint[] = {
|
| + {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ,
|
| + FTS5_BI_MATCH, 1, 1, -1},
|
| + {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ,
|
| + FTS5_BI_RANK, 2, 1, -1},
|
| + {SQLITE_INDEX_CONSTRAINT_EQ, FTS5_BI_ROWID_EQ, 0, 0, -1},
|
| + {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE,
|
| + FTS5_BI_ROWID_LE, 0, 0, -1},
|
| + {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE,
|
| + FTS5_BI_ROWID_GE, 0, 0, -1},
|
| + };
|
| +
|
| + int aColMap[3];
|
| + aColMap[0] = -1;
|
| + aColMap[1] = pConfig->nCol;
|
| + aColMap[2] = pConfig->nCol+1;
|
| +
|
| + /* Set idxFlags flags for all WHERE clause terms that will be used. */
|
| + for(i=0; i<pInfo->nConstraint; i++){
|
| + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
|
| + int j;
|
| + for(j=0; j<(int)ArraySize(aConstraint); j++){
|
| + struct Constraint *pC = &aConstraint[j];
|
| + if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){
|
| + if( p->usable ){
|
| + pC->iConsIndex = i;
|
| + idxFlags |= pC->fts5op;
|
| + }else if( j==0 ){
|
| + /* As there exists an unusable MATCH constraint this is an
|
| + ** unusable plan. Set a prohibitively high cost. */
|
| + pInfo->estimatedCost = 1e50;
|
| + return SQLITE_OK;
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + /* Set idxFlags flags for the ORDER BY clause */
|
| + if( pInfo->nOrderBy==1 ){
|
| + int iSort = pInfo->aOrderBy[0].iColumn;
|
| + if( iSort==(pConfig->nCol+1) && BitFlagTest(idxFlags, FTS5_BI_MATCH) ){
|
| + idxFlags |= FTS5_BI_ORDER_RANK;
|
| + }else if( iSort==-1 ){
|
| + idxFlags |= FTS5_BI_ORDER_ROWID;
|
| + }
|
| + if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){
|
| + pInfo->orderByConsumed = 1;
|
| + if( pInfo->aOrderBy[0].desc ){
|
| + idxFlags |= FTS5_BI_ORDER_DESC;
|
| + }
|
| + }
|
| + }
|
| +
|
| + /* Calculate the estimated cost based on the flags set in idxFlags. */
|
| + bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH);
|
| + if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){
|
| + pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0;
|
| + if( bHasMatch==0 ) fts5SetUniqueFlag(pInfo);
|
| + }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){
|
| + pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0;
|
| + }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){
|
| + pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0;
|
| + }else{
|
| + pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0;
|
| + }
|
| +
|
| + /* Assign argvIndex values to each constraint in use. */
|
| + iNext = 1;
|
| + for(i=0; i<(int)ArraySize(aConstraint); i++){
|
| + struct Constraint *pC = &aConstraint[i];
|
| + if( pC->iConsIndex>=0 ){
|
| + pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++;
|
| + pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit;
|
| + }
|
| + }
|
| +
|
| + pInfo->idxNum = idxFlags;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Implementation of xOpen method.
|
| +*/
|
| +static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
|
| + Fts5Table *pTab = (Fts5Table*)pVTab;
|
| + Fts5Config *pConfig = pTab->pConfig;
|
| + Fts5Cursor *pCsr; /* New cursor object */
|
| + int nByte; /* Bytes of space to allocate */
|
| + int rc = SQLITE_OK; /* Return code */
|
| +
|
| + nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
|
| + pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
|
| + if( pCsr ){
|
| + Fts5Global *pGlobal = pTab->pGlobal;
|
| + memset(pCsr, 0, nByte);
|
| + pCsr->aColumnSize = (int*)&pCsr[1];
|
| + pCsr->pNext = pGlobal->pCsr;
|
| + pGlobal->pCsr = pCsr;
|
| + pCsr->iCsrId = ++pGlobal->iNextId;
|
| + }else{
|
| + rc = SQLITE_NOMEM;
|
| + }
|
| + *ppCsr = (sqlite3_vtab_cursor*)pCsr;
|
| + return rc;
|
| +}
|
| +
|
| +static int fts5StmtType(Fts5Cursor *pCsr){
|
| + if( pCsr->ePlan==FTS5_PLAN_SCAN ){
|
| + return (pCsr->bDesc) ? FTS5_STMT_SCAN_DESC : FTS5_STMT_SCAN_ASC;
|
| + }
|
| + return FTS5_STMT_LOOKUP;
|
| +}
|
| +
|
| +/*
|
| +** This function is called after the cursor passed as the only argument
|
| +** is moved to point at a different row. It clears all cached data
|
| +** specific to the previous row stored by the cursor object.
|
| +*/
|
| +static void fts5CsrNewrow(Fts5Cursor *pCsr){
|
| + CsrFlagSet(pCsr,
|
| + FTS5CSR_REQUIRE_CONTENT
|
| + | FTS5CSR_REQUIRE_DOCSIZE
|
| + | FTS5CSR_REQUIRE_INST
|
| + );
|
| +}
|
| +
|
| +static void fts5FreeCursorComponents(Fts5Cursor *pCsr){
|
| + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
|
| + Fts5Auxdata *pData;
|
| + Fts5Auxdata *pNext;
|
| +
|
| + sqlite3_free(pCsr->aInstIter);
|
| + sqlite3_free(pCsr->aInst);
|
| + if( pCsr->pStmt ){
|
| + int eStmt = fts5StmtType(pCsr);
|
| + sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
|
| + }
|
| + if( pCsr->pSorter ){
|
| + Fts5Sorter *pSorter = pCsr->pSorter;
|
| + sqlite3_finalize(pSorter->pStmt);
|
| + sqlite3_free(pSorter);
|
| + }
|
| +
|
| + if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){
|
| + sqlite3Fts5ExprFree(pCsr->pExpr);
|
| + }
|
| +
|
| + for(pData=pCsr->pAuxdata; pData; pData=pNext){
|
| + pNext = pData->pNext;
|
| + if( pData->xDelete ) pData->xDelete(pData->pPtr);
|
| + sqlite3_free(pData);
|
| + }
|
| +
|
| + sqlite3_finalize(pCsr->pRankArgStmt);
|
| + sqlite3_free(pCsr->apRankArg);
|
| +
|
| + if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
|
| + sqlite3_free(pCsr->zRank);
|
| + sqlite3_free(pCsr->zRankArgs);
|
| + }
|
| +
|
| + memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr));
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Close the cursor. For additional information see the documentation
|
| +** on the xClose method of the virtual table interface.
|
| +*/
|
| +static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
|
| + if( pCursor ){
|
| + Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
|
| + Fts5Cursor **pp;
|
| +
|
| + fts5FreeCursorComponents(pCsr);
|
| + /* Remove the cursor from the Fts5Global.pCsr list */
|
| + for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
|
| + *pp = pCsr->pNext;
|
| +
|
| + sqlite3_free(pCsr);
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static int fts5SorterNext(Fts5Cursor *pCsr){
|
| + Fts5Sorter *pSorter = pCsr->pSorter;
|
| + int rc;
|
| +
|
| + rc = sqlite3_step(pSorter->pStmt);
|
| + if( rc==SQLITE_DONE ){
|
| + rc = SQLITE_OK;
|
| + CsrFlagSet(pCsr, FTS5CSR_EOF);
|
| + }else if( rc==SQLITE_ROW ){
|
| + const u8 *a;
|
| + const u8 *aBlob;
|
| + int nBlob;
|
| + int i;
|
| + int iOff = 0;
|
| + rc = SQLITE_OK;
|
| +
|
| + pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0);
|
| + nBlob = sqlite3_column_bytes(pSorter->pStmt, 1);
|
| + aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1);
|
| +
|
| + for(i=0; i<(pSorter->nIdx-1); i++){
|
| + int iVal;
|
| + a += fts5GetVarint32(a, iVal);
|
| + iOff += iVal;
|
| + pSorter->aIdx[i] = iOff;
|
| + }
|
| + pSorter->aIdx[i] = &aBlob[nBlob] - a;
|
| +
|
| + pSorter->aPoslist = a;
|
| + fts5CsrNewrow(pCsr);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Set the FTS5CSR_REQUIRE_RESEEK flag on all FTS5_PLAN_MATCH cursors
|
| +** open on table pTab.
|
| +*/
|
| +static void fts5TripCursors(Fts5Table *pTab){
|
| + Fts5Cursor *pCsr;
|
| + for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
|
| + if( pCsr->ePlan==FTS5_PLAN_MATCH
|
| + && pCsr->base.pVtab==(sqlite3_vtab*)pTab
|
| + ){
|
| + CsrFlagSet(pCsr, FTS5CSR_REQUIRE_RESEEK);
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** If the REQUIRE_RESEEK flag is set on the cursor passed as the first
|
| +** argument, close and reopen all Fts5IndexIter iterators that the cursor
|
| +** is using. Then attempt to move the cursor to a rowid equal to or laster
|
| +** (in the cursors sort order - ASC or DESC) than the current rowid.
|
| +**
|
| +** If the new rowid is not equal to the old, set output parameter *pbSkip
|
| +** to 1 before returning. Otherwise, leave it unchanged.
|
| +**
|
| +** Return SQLITE_OK if successful or if no reseek was required, or an
|
| +** error code if an error occurred.
|
| +*/
|
| +static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
|
| + int rc = SQLITE_OK;
|
| + assert( *pbSkip==0 );
|
| + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_RESEEK) ){
|
| + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
|
| + int bDesc = pCsr->bDesc;
|
| + i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
|
| +
|
| + rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, iRowid, bDesc);
|
| + if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){
|
| + *pbSkip = 1;
|
| + }
|
| +
|
| + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK);
|
| + fts5CsrNewrow(pCsr);
|
| + if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
|
| + CsrFlagSet(pCsr, FTS5CSR_EOF);
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Advance the cursor to the next row in the table that matches the
|
| +** search criteria.
|
| +**
|
| +** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
|
| +** even if we reach end-of-file. The fts5EofMethod() will be called
|
| +** subsequently to determine whether or not an EOF was hit.
|
| +*/
|
| +static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
|
| + int rc = SQLITE_OK;
|
| +
|
| + assert( (pCsr->ePlan<3)==
|
| + (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE)
|
| + );
|
| +
|
| + if( pCsr->ePlan<3 ){
|
| + int bSkip = 0;
|
| + if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
|
| + rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
|
| + if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
|
| + CsrFlagSet(pCsr, FTS5CSR_EOF);
|
| + }
|
| + fts5CsrNewrow(pCsr);
|
| + }else{
|
| + switch( pCsr->ePlan ){
|
| + case FTS5_PLAN_SPECIAL: {
|
| + CsrFlagSet(pCsr, FTS5CSR_EOF);
|
| + break;
|
| + }
|
| +
|
| + case FTS5_PLAN_SORTED_MATCH: {
|
| + rc = fts5SorterNext(pCsr);
|
| + break;
|
| + }
|
| +
|
| + default:
|
| + rc = sqlite3_step(pCsr->pStmt);
|
| + if( rc!=SQLITE_ROW ){
|
| + CsrFlagSet(pCsr, FTS5CSR_EOF);
|
| + rc = sqlite3_reset(pCsr->pStmt);
|
| + }else{
|
| + rc = SQLITE_OK;
|
| + }
|
| + break;
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +static sqlite3_stmt *fts5PrepareStatement(
|
| + int *pRc,
|
| + Fts5Config *pConfig,
|
| + const char *zFmt,
|
| + ...
|
| +){
|
| + sqlite3_stmt *pRet = 0;
|
| + va_list ap;
|
| + va_start(ap, zFmt);
|
| +
|
| + if( *pRc==SQLITE_OK ){
|
| + int rc;
|
| + char *zSql = sqlite3_vmprintf(zFmt, ap);
|
| + if( zSql==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
|
| + if( rc!=SQLITE_OK ){
|
| + *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
|
| + }
|
| + sqlite3_free(zSql);
|
| + }
|
| + *pRc = rc;
|
| + }
|
| +
|
| + va_end(ap);
|
| + return pRet;
|
| +}
|
| +
|
| +static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
|
| + Fts5Config *pConfig = pTab->pConfig;
|
| + Fts5Sorter *pSorter;
|
| + int nPhrase;
|
| + int nByte;
|
| + int rc = SQLITE_OK;
|
| + const char *zRank = pCsr->zRank;
|
| + const char *zRankArgs = pCsr->zRankArgs;
|
| +
|
| + nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
|
| + nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1);
|
| + pSorter = (Fts5Sorter*)sqlite3_malloc(nByte);
|
| + if( pSorter==0 ) return SQLITE_NOMEM;
|
| + memset(pSorter, 0, nByte);
|
| + pSorter->nIdx = nPhrase;
|
| +
|
| + /* TODO: It would be better to have some system for reusing statement
|
| + ** handles here, rather than preparing a new one for each query. But that
|
| + ** is not possible as SQLite reference counts the virtual table objects.
|
| + ** And since the statement required here reads from this very virtual
|
| + ** table, saving it creates a circular reference.
|
| + **
|
| + ** If SQLite a built-in statement cache, this wouldn't be a problem. */
|
| + pSorter->pStmt = fts5PrepareStatement(&rc, pConfig,
|
| + "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s",
|
| + pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
|
| + (zRankArgs ? ", " : ""),
|
| + (zRankArgs ? zRankArgs : ""),
|
| + bDesc ? "DESC" : "ASC"
|
| + );
|
| +
|
| + pCsr->pSorter = pSorter;
|
| + if( rc==SQLITE_OK ){
|
| + assert( pTab->pSortCsr==0 );
|
| + pTab->pSortCsr = pCsr;
|
| + rc = fts5SorterNext(pCsr);
|
| + pTab->pSortCsr = 0;
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + sqlite3_finalize(pSorter->pStmt);
|
| + sqlite3_free(pSorter);
|
| + pCsr->pSorter = 0;
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
|
| + int rc;
|
| + Fts5Expr *pExpr = pCsr->pExpr;
|
| + rc = sqlite3Fts5ExprFirst(pExpr, pTab->pIndex, pCsr->iFirstRowid, bDesc);
|
| + if( sqlite3Fts5ExprEof(pExpr) ){
|
| + CsrFlagSet(pCsr, FTS5CSR_EOF);
|
| + }
|
| + fts5CsrNewrow(pCsr);
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Process a "special" query. A special query is identified as one with a
|
| +** MATCH expression that begins with a '*' character. The remainder of
|
| +** the text passed to the MATCH operator are used as the special query
|
| +** parameters.
|
| +*/
|
| +static int fts5SpecialMatch(
|
| + Fts5Table *pTab,
|
| + Fts5Cursor *pCsr,
|
| + const char *zQuery
|
| +){
|
| + int rc = SQLITE_OK; /* Return code */
|
| + const char *z = zQuery; /* Special query text */
|
| + int n; /* Number of bytes in text at z */
|
| +
|
| + while( z[0]==' ' ) z++;
|
| + for(n=0; z[n] && z[n]!=' '; n++);
|
| +
|
| + assert( pTab->base.zErrMsg==0 );
|
| + pCsr->ePlan = FTS5_PLAN_SPECIAL;
|
| +
|
| + if( 0==sqlite3_strnicmp("reads", z, n) ){
|
| + pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->pIndex);
|
| + }
|
| + else if( 0==sqlite3_strnicmp("id", z, n) ){
|
| + pCsr->iSpecial = pCsr->iCsrId;
|
| + }
|
| + else{
|
| + /* An unrecognized directive. Return an error message. */
|
| + pTab->base.zErrMsg = sqlite3_mprintf("unknown special query: %.*s", n, z);
|
| + rc = SQLITE_ERROR;
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Search for an auxiliary function named zName that can be used with table
|
| +** pTab. If one is found, return a pointer to the corresponding Fts5Auxiliary
|
| +** structure. Otherwise, if no such function exists, return NULL.
|
| +*/
|
| +static Fts5Auxiliary *fts5FindAuxiliary(Fts5Table *pTab, const char *zName){
|
| + Fts5Auxiliary *pAux;
|
| +
|
| + for(pAux=pTab->pGlobal->pAux; pAux; pAux=pAux->pNext){
|
| + if( sqlite3_stricmp(zName, pAux->zFunc)==0 ) return pAux;
|
| + }
|
| +
|
| + /* No function of the specified name was found. Return 0. */
|
| + return 0;
|
| +}
|
| +
|
| +
|
| +static int fts5FindRankFunction(Fts5Cursor *pCsr){
|
| + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
|
| + Fts5Config *pConfig = pTab->pConfig;
|
| + int rc = SQLITE_OK;
|
| + Fts5Auxiliary *pAux = 0;
|
| + const char *zRank = pCsr->zRank;
|
| + const char *zRankArgs = pCsr->zRankArgs;
|
| +
|
| + if( zRankArgs ){
|
| + char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs);
|
| + if( zSql ){
|
| + sqlite3_stmt *pStmt = 0;
|
| + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pStmt, 0);
|
| + sqlite3_free(zSql);
|
| + assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 );
|
| + if( rc==SQLITE_OK ){
|
| + if( SQLITE_ROW==sqlite3_step(pStmt) ){
|
| + int nByte;
|
| + pCsr->nRankArg = sqlite3_column_count(pStmt);
|
| + nByte = sizeof(sqlite3_value*)*pCsr->nRankArg;
|
| + pCsr->apRankArg = (sqlite3_value**)sqlite3Fts5MallocZero(&rc, nByte);
|
| + if( rc==SQLITE_OK ){
|
| + int i;
|
| + for(i=0; i<pCsr->nRankArg; i++){
|
| + pCsr->apRankArg[i] = sqlite3_column_value(pStmt, i);
|
| + }
|
| + }
|
| + pCsr->pRankArgStmt = pStmt;
|
| + }else{
|
| + rc = sqlite3_finalize(pStmt);
|
| + assert( rc!=SQLITE_OK );
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + pAux = fts5FindAuxiliary(pTab, zRank);
|
| + if( pAux==0 ){
|
| + assert( pTab->base.zErrMsg==0 );
|
| + pTab->base.zErrMsg = sqlite3_mprintf("no such function: %s", zRank);
|
| + rc = SQLITE_ERROR;
|
| + }
|
| + }
|
| +
|
| + pCsr->pRank = pAux;
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +static int fts5CursorParseRank(
|
| + Fts5Config *pConfig,
|
| + Fts5Cursor *pCsr,
|
| + sqlite3_value *pRank
|
| +){
|
| + int rc = SQLITE_OK;
|
| + if( pRank ){
|
| + const char *z = (const char*)sqlite3_value_text(pRank);
|
| + char *zRank = 0;
|
| + char *zRankArgs = 0;
|
| +
|
| + if( z==0 ){
|
| + if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR;
|
| + }else{
|
| + rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + pCsr->zRank = zRank;
|
| + pCsr->zRankArgs = zRankArgs;
|
| + CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK);
|
| + }else if( rc==SQLITE_ERROR ){
|
| + pCsr->base.pVtab->zErrMsg = sqlite3_mprintf(
|
| + "parse error in rank function: %s", z
|
| + );
|
| + }
|
| + }else{
|
| + if( pConfig->zRank ){
|
| + pCsr->zRank = (char*)pConfig->zRank;
|
| + pCsr->zRankArgs = (char*)pConfig->zRankArgs;
|
| + }else{
|
| + pCsr->zRank = (char*)FTS5_DEFAULT_RANK;
|
| + pCsr->zRankArgs = 0;
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){
|
| + if( pVal ){
|
| + int eType = sqlite3_value_numeric_type(pVal);
|
| + if( eType==SQLITE_INTEGER ){
|
| + return sqlite3_value_int64(pVal);
|
| + }
|
| + }
|
| + return iDefault;
|
| +}
|
| +
|
| +/*
|
| +** This is the xFilter interface for the virtual table. See
|
| +** the virtual table xFilter method documentation for additional
|
| +** information.
|
| +**
|
| +** There are three possible query strategies:
|
| +**
|
| +** 1. Full-text search using a MATCH operator.
|
| +** 2. A by-rowid lookup.
|
| +** 3. A full-table scan.
|
| +*/
|
| +static int fts5FilterMethod(
|
| + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
|
| + int idxNum, /* Strategy index */
|
| + const char *idxStr, /* Unused */
|
| + int nVal, /* Number of elements in apVal */
|
| + sqlite3_value **apVal /* Arguments for the indexing scheme */
|
| +){
|
| + Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
|
| + Fts5Config *pConfig = pTab->pConfig;
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
|
| + int rc = SQLITE_OK; /* Error code */
|
| + int iVal = 0; /* Counter for apVal[] */
|
| + int bDesc; /* True if ORDER BY [rank|rowid] DESC */
|
| + int bOrderByRank; /* True if ORDER BY rank */
|
| + sqlite3_value *pMatch = 0; /* <tbl> MATCH ? expression (or NULL) */
|
| + sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */
|
| + sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */
|
| + sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */
|
| + sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */
|
| + char **pzErrmsg = pConfig->pzErrmsg;
|
| +
|
| + if( pCsr->ePlan ){
|
| + fts5FreeCursorComponents(pCsr);
|
| + memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr));
|
| + }
|
| +
|
| + assert( pCsr->pStmt==0 );
|
| + assert( pCsr->pExpr==0 );
|
| + assert( pCsr->csrflags==0 );
|
| + assert( pCsr->pRank==0 );
|
| + assert( pCsr->zRank==0 );
|
| + assert( pCsr->zRankArgs==0 );
|
| +
|
| + assert( pzErrmsg==0 || pzErrmsg==&pTab->base.zErrMsg );
|
| + pConfig->pzErrmsg = &pTab->base.zErrMsg;
|
| +
|
| + /* Decode the arguments passed through to this function.
|
| + **
|
| + ** Note: The following set of if(...) statements must be in the same
|
| + ** order as the corresponding entries in the struct at the top of
|
| + ** fts5BestIndexMethod(). */
|
| + if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++];
|
| + if( BitFlagTest(idxNum, FTS5_BI_RANK) ) pRank = apVal[iVal++];
|
| + if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++];
|
| + if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++];
|
| + if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++];
|
| + assert( iVal==nVal );
|
| + bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0);
|
| + pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0);
|
| +
|
| + /* Set the cursor upper and lower rowid limits. Only some strategies
|
| + ** actually use them. This is ok, as the xBestIndex() method leaves the
|
| + ** sqlite3_index_constraint.omit flag clear for range constraints
|
| + ** on the rowid field. */
|
| + if( pRowidEq ){
|
| + pRowidLe = pRowidGe = pRowidEq;
|
| + }
|
| + if( bDesc ){
|
| + pCsr->iFirstRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64);
|
| + pCsr->iLastRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64);
|
| + }else{
|
| + pCsr->iLastRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64);
|
| + pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64);
|
| + }
|
| +
|
| + if( pTab->pSortCsr ){
|
| + /* If pSortCsr is non-NULL, then this call is being made as part of
|
| + ** processing for a "... MATCH <expr> ORDER BY rank" query (ePlan is
|
| + ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will
|
| + ** return results to the user for this query. The current cursor
|
| + ** (pCursor) is used to execute the query issued by function
|
| + ** fts5CursorFirstSorted() above. */
|
| + assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 );
|
| + assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
|
| + assert( pCsr->iLastRowid==LARGEST_INT64 );
|
| + assert( pCsr->iFirstRowid==SMALLEST_INT64 );
|
| + pCsr->ePlan = FTS5_PLAN_SOURCE;
|
| + pCsr->pExpr = pTab->pSortCsr->pExpr;
|
| + rc = fts5CursorFirst(pTab, pCsr, bDesc);
|
| + }else if( pMatch ){
|
| + const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
|
| + if( zExpr==0 ) zExpr = "";
|
| +
|
| + rc = fts5CursorParseRank(pConfig, pCsr, pRank);
|
| + if( rc==SQLITE_OK ){
|
| + if( zExpr[0]=='*' ){
|
| + /* The user has issued a query of the form "MATCH '*...'". This
|
| + ** indicates that the MATCH expression is not a full text query,
|
| + ** but a request for an internal parameter. */
|
| + rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]);
|
| + }else{
|
| + char **pzErr = &pTab->base.zErrMsg;
|
| + rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr);
|
| + if( rc==SQLITE_OK ){
|
| + if( bOrderByRank ){
|
| + pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
|
| + rc = fts5CursorFirstSorted(pTab, pCsr, bDesc);
|
| + }else{
|
| + pCsr->ePlan = FTS5_PLAN_MATCH;
|
| + rc = fts5CursorFirst(pTab, pCsr, bDesc);
|
| + }
|
| + }
|
| + }
|
| + }
|
| + }else if( pConfig->zContent==0 ){
|
| + *pConfig->pzErrmsg = sqlite3_mprintf(
|
| + "%s: table does not support scanning", pConfig->zName
|
| + );
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup
|
| + ** by rowid (ePlan==FTS5_PLAN_ROWID). */
|
| + pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN);
|
| + rc = sqlite3Fts5StorageStmt(
|
| + pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->base.zErrMsg
|
| + );
|
| + if( rc==SQLITE_OK ){
|
| + if( pCsr->ePlan==FTS5_PLAN_ROWID ){
|
| + sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
|
| + }else{
|
| + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid);
|
| + sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid);
|
| + }
|
| + rc = fts5NextMethod(pCursor);
|
| + }
|
| + }
|
| +
|
| + pConfig->pzErrmsg = pzErrmsg;
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** This is the xEof method of the virtual table. SQLite calls this
|
| +** routine to find out if it has reached the end of a result set.
|
| +*/
|
| +static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
|
| + return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0);
|
| +}
|
| +
|
| +/*
|
| +** Return the rowid that the cursor currently points to.
|
| +*/
|
| +static i64 fts5CursorRowid(Fts5Cursor *pCsr){
|
| + assert( pCsr->ePlan==FTS5_PLAN_MATCH
|
| + || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH
|
| + || pCsr->ePlan==FTS5_PLAN_SOURCE
|
| + );
|
| + if( pCsr->pSorter ){
|
| + return pCsr->pSorter->iRowid;
|
| + }else{
|
| + return sqlite3Fts5ExprRowid(pCsr->pExpr);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** This is the xRowid method. The SQLite core calls this routine to
|
| +** retrieve the rowid for the current row of the result set. fts5
|
| +** exposes %_content.rowid as the rowid for the virtual table. The
|
| +** rowid should be written to *pRowid.
|
| +*/
|
| +static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
|
| + int ePlan = pCsr->ePlan;
|
| +
|
| + assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
|
| + switch( ePlan ){
|
| + case FTS5_PLAN_SPECIAL:
|
| + *pRowid = 0;
|
| + break;
|
| +
|
| + case FTS5_PLAN_SOURCE:
|
| + case FTS5_PLAN_MATCH:
|
| + case FTS5_PLAN_SORTED_MATCH:
|
| + *pRowid = fts5CursorRowid(pCsr);
|
| + break;
|
| +
|
| + default:
|
| + *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
|
| + break;
|
| + }
|
| +
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** If the cursor requires seeking (bSeekRequired flag is set), seek it.
|
| +** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise.
|
| +**
|
| +** If argument bErrormsg is true and an error occurs, an error message may
|
| +** be left in sqlite3_vtab.zErrMsg.
|
| +*/
|
| +static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){
|
| + int rc = SQLITE_OK;
|
| +
|
| + /* If the cursor does not yet have a statement handle, obtain one now. */
|
| + if( pCsr->pStmt==0 ){
|
| + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
|
| + int eStmt = fts5StmtType(pCsr);
|
| + rc = sqlite3Fts5StorageStmt(
|
| + pTab->pStorage, eStmt, &pCsr->pStmt, (bErrormsg?&pTab->base.zErrMsg:0)
|
| + );
|
| + assert( rc!=SQLITE_OK || pTab->base.zErrMsg==0 );
|
| + assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) );
|
| + }
|
| +
|
| + if( rc==SQLITE_OK && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){
|
| + assert( pCsr->pExpr );
|
| + sqlite3_reset(pCsr->pStmt);
|
| + sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr));
|
| + rc = sqlite3_step(pCsr->pStmt);
|
| + if( rc==SQLITE_ROW ){
|
| + rc = SQLITE_OK;
|
| + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT);
|
| + }else{
|
| + rc = sqlite3_reset(pCsr->pStmt);
|
| + if( rc==SQLITE_OK ){
|
| + rc = FTS5_CORRUPT;
|
| + }
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static void fts5SetVtabError(Fts5Table *p, const char *zFormat, ...){
|
| + va_list ap; /* ... printf arguments */
|
| + va_start(ap, zFormat);
|
| + assert( p->base.zErrMsg==0 );
|
| + p->base.zErrMsg = sqlite3_vmprintf(zFormat, ap);
|
| + va_end(ap);
|
| +}
|
| +
|
| +/*
|
| +** This function is called to handle an FTS INSERT command. In other words,
|
| +** an INSERT statement of the form:
|
| +**
|
| +** INSERT INTO fts(fts) VALUES($pCmd)
|
| +** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal)
|
| +**
|
| +** Argument pVal is the value assigned to column "fts" by the INSERT
|
| +** statement. This function returns SQLITE_OK if successful, or an SQLite
|
| +** error code if an error occurs.
|
| +**
|
| +** The commands implemented by this function are documented in the "Special
|
| +** INSERT Directives" section of the documentation. It should be updated if
|
| +** more commands are added to this function.
|
| +*/
|
| +static int fts5SpecialInsert(
|
| + Fts5Table *pTab, /* Fts5 table object */
|
| + const char *zCmd, /* Text inserted into table-name column */
|
| + sqlite3_value *pVal /* Value inserted into rank column */
|
| +){
|
| + Fts5Config *pConfig = pTab->pConfig;
|
| + int rc = SQLITE_OK;
|
| + int bError = 0;
|
| +
|
| + if( 0==sqlite3_stricmp("delete-all", zCmd) ){
|
| + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
|
| + fts5SetVtabError(pTab,
|
| + "'delete-all' may only be used with a "
|
| + "contentless or external content fts5 table"
|
| + );
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
|
| + }
|
| + }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){
|
| + if( pConfig->eContent==FTS5_CONTENT_NONE ){
|
| + fts5SetVtabError(pTab,
|
| + "'rebuild' may not be used with a contentless fts5 table"
|
| + );
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
|
| + }
|
| + }else if( 0==sqlite3_stricmp("optimize", zCmd) ){
|
| + rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
|
| + }else if( 0==sqlite3_stricmp("merge", zCmd) ){
|
| + int nMerge = sqlite3_value_int(pVal);
|
| + rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge);
|
| + }else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){
|
| + rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
|
| +#ifdef SQLITE_DEBUG
|
| + }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){
|
| + pConfig->bPrefixIndex = sqlite3_value_int(pVal);
|
| +#endif
|
| + }else{
|
| + rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex);
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, zCmd, pVal, &bError);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + if( bError ){
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, zCmd, pVal, 0);
|
| + }
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static int fts5SpecialDelete(
|
| + Fts5Table *pTab,
|
| + sqlite3_value **apVal,
|
| + sqlite3_int64 *piRowid
|
| +){
|
| + int rc = SQLITE_OK;
|
| + int eType1 = sqlite3_value_type(apVal[1]);
|
| + if( eType1==SQLITE_INTEGER ){
|
| + sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
|
| + rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static void fts5StorageInsert(
|
| + int *pRc,
|
| + Fts5Table *pTab,
|
| + sqlite3_value **apVal,
|
| + i64 *piRowid
|
| +){
|
| + int rc = *pRc;
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, piRowid);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid);
|
| + }
|
| + *pRc = rc;
|
| +}
|
| +
|
| +/*
|
| +** This function is the implementation of the xUpdate callback used by
|
| +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
|
| +** inserted, updated or deleted.
|
| +**
|
| +** A delete specifies a single argument - the rowid of the row to remove.
|
| +**
|
| +** Update and insert operations pass:
|
| +**
|
| +** 1. The "old" rowid, or NULL.
|
| +** 2. The "new" rowid.
|
| +** 3. Values for each of the nCol matchable columns.
|
| +** 4. Values for the two hidden columns (<tablename> and "rank").
|
| +*/
|
| +static int fts5UpdateMethod(
|
| + sqlite3_vtab *pVtab, /* Virtual table handle */
|
| + int nArg, /* Size of argument array */
|
| + sqlite3_value **apVal, /* Array of arguments */
|
| + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
|
| +){
|
| + Fts5Table *pTab = (Fts5Table*)pVtab;
|
| + Fts5Config *pConfig = pTab->pConfig;
|
| + int eType0; /* value_type() of apVal[0] */
|
| + int rc = SQLITE_OK; /* Return code */
|
| +
|
| + /* A transaction must be open when this is called. */
|
| + assert( pTab->ts.eState==1 );
|
| +
|
| + assert( pVtab->zErrMsg==0 );
|
| + assert( nArg==1 || nArg==(2+pConfig->nCol+2) );
|
| + assert( nArg==1
|
| + || sqlite3_value_type(apVal[1])==SQLITE_INTEGER
|
| + || sqlite3_value_type(apVal[1])==SQLITE_NULL
|
| + );
|
| + assert( pTab->pConfig->pzErrmsg==0 );
|
| + pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg;
|
| +
|
| + /* Put any active cursors into REQUIRE_SEEK state. */
|
| + fts5TripCursors(pTab);
|
| +
|
| + eType0 = sqlite3_value_type(apVal[0]);
|
| + if( eType0==SQLITE_NULL
|
| + && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL
|
| + ){
|
| + /* A "special" INSERT op. These are handled separately. */
|
| + const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]);
|
| + if( pConfig->eContent!=FTS5_CONTENT_NORMAL
|
| + && 0==sqlite3_stricmp("delete", z)
|
| + ){
|
| + rc = fts5SpecialDelete(pTab, apVal, pRowid);
|
| + }else{
|
| + rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
|
| + }
|
| + }else{
|
| + /* A regular INSERT, UPDATE or DELETE statement. The trick here is that
|
| + ** any conflict on the rowid value must be detected before any
|
| + ** modifications are made to the database file. There are 4 cases:
|
| + **
|
| + ** 1) DELETE
|
| + ** 2) UPDATE (rowid not modified)
|
| + ** 3) UPDATE (rowid modified)
|
| + ** 4) INSERT
|
| + **
|
| + ** Cases 3 and 4 may violate the rowid constraint.
|
| + */
|
| + int eConflict = SQLITE_ABORT;
|
| + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
|
| + eConflict = sqlite3_vtab_on_conflict(pConfig->db);
|
| + }
|
| +
|
| + assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
|
| + assert( nArg!=1 || eType0==SQLITE_INTEGER );
|
| +
|
| + /* Filter out attempts to run UPDATE or DELETE on contentless tables.
|
| + ** This is not suported. */
|
| + if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){
|
| + pTab->base.zErrMsg = sqlite3_mprintf(
|
| + "cannot %s contentless fts5 table: %s",
|
| + (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
|
| + );
|
| + rc = SQLITE_ERROR;
|
| + }
|
| +
|
| + /* Case 1: DELETE */
|
| + else if( nArg==1 ){
|
| + i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */
|
| + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
|
| + }
|
| +
|
| + /* Case 2: INSERT */
|
| + else if( eType0!=SQLITE_INTEGER ){
|
| + /* If this is a REPLACE, first remove the current entry (if any) */
|
| + if( eConflict==SQLITE_REPLACE
|
| + && sqlite3_value_type(apVal[1])==SQLITE_INTEGER
|
| + ){
|
| + i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */
|
| + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
|
| + }
|
| + fts5StorageInsert(&rc, pTab, apVal, pRowid);
|
| + }
|
| +
|
| + /* Case 2: UPDATE */
|
| + else{
|
| + i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */
|
| + i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */
|
| + if( iOld!=iNew ){
|
| + if( eConflict==SQLITE_REPLACE ){
|
| + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
|
| + }
|
| + fts5StorageInsert(&rc, pTab, apVal, pRowid);
|
| + }else{
|
| + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid);
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid);
|
| + }
|
| + }
|
| + }else{
|
| + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
|
| + fts5StorageInsert(&rc, pTab, apVal, pRowid);
|
| + }
|
| + }
|
| + }
|
| +
|
| + pTab->pConfig->pzErrmsg = 0;
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Implementation of xSync() method.
|
| +*/
|
| +static int fts5SyncMethod(sqlite3_vtab *pVtab){
|
| + int rc;
|
| + Fts5Table *pTab = (Fts5Table*)pVtab;
|
| + fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
|
| + pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg;
|
| + fts5TripCursors(pTab);
|
| + rc = sqlite3Fts5StorageSync(pTab->pStorage, 1);
|
| + pTab->pConfig->pzErrmsg = 0;
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Implementation of xBegin() method.
|
| +*/
|
| +static int fts5BeginMethod(sqlite3_vtab *pVtab){
|
| + fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Implementation of xCommit() method. This is a no-op. The contents of
|
| +** the pending-terms hash-table have already been flushed into the database
|
| +** by fts5SyncMethod().
|
| +*/
|
| +static int fts5CommitMethod(sqlite3_vtab *pVtab){
|
| + fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_COMMIT, 0);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Implementation of xRollback(). Discard the contents of the pending-terms
|
| +** hash-table. Any changes made to the database are reverted by SQLite.
|
| +*/
|
| +static int fts5RollbackMethod(sqlite3_vtab *pVtab){
|
| + int rc;
|
| + Fts5Table *pTab = (Fts5Table*)pVtab;
|
| + fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0);
|
| + rc = sqlite3Fts5StorageRollback(pTab->pStorage);
|
| + return rc;
|
| +}
|
| +
|
| +static void *fts5ApiUserData(Fts5Context *pCtx){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + return pCsr->pAux->pUserData;
|
| +}
|
| +
|
| +static int fts5ApiColumnCount(Fts5Context *pCtx){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol;
|
| +}
|
| +
|
| +static int fts5ApiColumnTotalSize(
|
| + Fts5Context *pCtx,
|
| + int iCol,
|
| + sqlite3_int64 *pnToken
|
| +){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
|
| + return sqlite3Fts5StorageSize(pTab->pStorage, iCol, pnToken);
|
| +}
|
| +
|
| +static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
|
| + return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow);
|
| +}
|
| +
|
| +static int fts5ApiTokenize(
|
| + Fts5Context *pCtx,
|
| + const char *pText, int nText,
|
| + void *pUserData,
|
| + int (*xToken)(void*, int, const char*, int, int, int)
|
| +){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
|
| + return sqlite3Fts5Tokenize(
|
| + pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken
|
| + );
|
| +}
|
| +
|
| +static int fts5ApiPhraseCount(Fts5Context *pCtx){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + return sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
|
| +}
|
| +
|
| +static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
|
| +}
|
| +
|
| +static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){
|
| + int n;
|
| + if( pCsr->pSorter ){
|
| + Fts5Sorter *pSorter = pCsr->pSorter;
|
| + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
|
| + n = pSorter->aIdx[iPhrase] - i1;
|
| + *pa = &pSorter->aPoslist[i1];
|
| + }else{
|
| + n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
|
| + }
|
| + return n;
|
| +}
|
| +
|
| +/*
|
| +** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated
|
| +** correctly for the current view. Return SQLITE_OK if successful, or an
|
| +** SQLite error code otherwise.
|
| +*/
|
| +static int fts5CacheInstArray(Fts5Cursor *pCsr){
|
| + int rc = SQLITE_OK;
|
| + Fts5PoslistReader *aIter; /* One iterator for each phrase */
|
| + int nIter; /* Number of iterators/phrases */
|
| +
|
| + nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
|
| + if( pCsr->aInstIter==0 ){
|
| + int nByte = sizeof(Fts5PoslistReader) * nIter;
|
| + pCsr->aInstIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte);
|
| + }
|
| + aIter = pCsr->aInstIter;
|
| +
|
| + if( aIter ){
|
| + int nInst = 0; /* Number instances seen so far */
|
| + int i;
|
| +
|
| + /* Initialize all iterators */
|
| + for(i=0; i<nIter; i++){
|
| + const u8 *a;
|
| + int n = fts5CsrPoslist(pCsr, i, &a);
|
| + sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
|
| + }
|
| +
|
| + while( 1 ){
|
| + int *aInst;
|
| + int iBest = -1;
|
| + for(i=0; i<nIter; i++){
|
| + if( (aIter[i].bEof==0)
|
| + && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos)
|
| + ){
|
| + iBest = i;
|
| + }
|
| + }
|
| + if( iBest<0 ) break;
|
| +
|
| + nInst++;
|
| + if( nInst>=pCsr->nInstAlloc ){
|
| + pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
|
| + aInst = (int*)sqlite3_realloc(
|
| + pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3
|
| + );
|
| + if( aInst ){
|
| + pCsr->aInst = aInst;
|
| + }else{
|
| + rc = SQLITE_NOMEM;
|
| + break;
|
| + }
|
| + }
|
| +
|
| + aInst = &pCsr->aInst[3 * (nInst-1)];
|
| + aInst[0] = iBest;
|
| + aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos);
|
| + aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos);
|
| + sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
|
| + }
|
| +
|
| + pCsr->nInstCount = nInst;
|
| + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + int rc = SQLITE_OK;
|
| + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0
|
| + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){
|
| + *pnInst = pCsr->nInstCount;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static int fts5ApiInst(
|
| + Fts5Context *pCtx,
|
| + int iIdx,
|
| + int *piPhrase,
|
| + int *piCol,
|
| + int *piOff
|
| +){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + int rc = SQLITE_OK;
|
| + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0
|
| + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr))
|
| + ){
|
| + if( iIdx<0 || iIdx>=pCsr->nInstCount ){
|
| + rc = SQLITE_RANGE;
|
| + }else{
|
| + *piPhrase = pCsr->aInst[iIdx*3];
|
| + *piCol = pCsr->aInst[iIdx*3 + 1];
|
| + *piOff = pCsr->aInst[iIdx*3 + 2];
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
|
| + return fts5CursorRowid((Fts5Cursor*)pCtx);
|
| +}
|
| +
|
| +static int fts5ApiColumnText(
|
| + Fts5Context *pCtx,
|
| + int iCol,
|
| + const char **pz,
|
| + int *pn
|
| +){
|
| + int rc = SQLITE_OK;
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
|
| + *pz = 0;
|
| + *pn = 0;
|
| + }else{
|
| + rc = fts5SeekCursor(pCsr, 0);
|
| + if( rc==SQLITE_OK ){
|
| + *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
|
| + *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static int fts5ColumnSizeCb(
|
| + void *pContext, /* Pointer to int */
|
| + int tflags,
|
| + const char *pToken, /* Buffer containing token */
|
| + int nToken, /* Size of token in bytes */
|
| + int iStart, /* Start offset of token */
|
| + int iEnd /* End offset of token */
|
| +){
|
| + int *pCnt = (int*)pContext;
|
| + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
|
| + (*pCnt)++;
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
|
| + Fts5Config *pConfig = pTab->pConfig;
|
| + int rc = SQLITE_OK;
|
| +
|
| + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
|
| + if( pConfig->bColumnsize ){
|
| + i64 iRowid = fts5CursorRowid(pCsr);
|
| + rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
|
| + }else if( pConfig->zContent==0 ){
|
| + int i;
|
| + for(i=0; i<pConfig->nCol; i++){
|
| + if( pConfig->abUnindexed[i]==0 ){
|
| + pCsr->aColumnSize[i] = -1;
|
| + }
|
| + }
|
| + }else{
|
| + int i;
|
| + for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
|
| + if( pConfig->abUnindexed[i]==0 ){
|
| + const char *z; int n;
|
| + void *p = (void*)(&pCsr->aColumnSize[i]);
|
| + pCsr->aColumnSize[i] = 0;
|
| + rc = fts5ApiColumnText(pCtx, i, &z, &n);
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5Tokenize(
|
| + pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb
|
| + );
|
| + }
|
| + }
|
| + }
|
| + }
|
| + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
|
| + }
|
| + if( iCol<0 ){
|
| + int i;
|
| + *pnToken = 0;
|
| + for(i=0; i<pConfig->nCol; i++){
|
| + *pnToken += pCsr->aColumnSize[i];
|
| + }
|
| + }else if( iCol<pConfig->nCol ){
|
| + *pnToken = pCsr->aColumnSize[iCol];
|
| + }else{
|
| + *pnToken = 0;
|
| + rc = SQLITE_RANGE;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Implementation of the xSetAuxdata() method.
|
| +*/
|
| +static int fts5ApiSetAuxdata(
|
| + Fts5Context *pCtx, /* Fts5 context */
|
| + void *pPtr, /* Pointer to save as auxdata */
|
| + void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */
|
| +){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + Fts5Auxdata *pData;
|
| +
|
| + /* Search through the cursors list of Fts5Auxdata objects for one that
|
| + ** corresponds to the currently executing auxiliary function. */
|
| + for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){
|
| + if( pData->pAux==pCsr->pAux ) break;
|
| + }
|
| +
|
| + if( pData ){
|
| + if( pData->xDelete ){
|
| + pData->xDelete(pData->pPtr);
|
| + }
|
| + }else{
|
| + int rc = SQLITE_OK;
|
| + pData = (Fts5Auxdata*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Auxdata));
|
| + if( pData==0 ){
|
| + if( xDelete ) xDelete(pPtr);
|
| + return rc;
|
| + }
|
| + pData->pAux = pCsr->pAux;
|
| + pData->pNext = pCsr->pAuxdata;
|
| + pCsr->pAuxdata = pData;
|
| + }
|
| +
|
| + pData->xDelete = xDelete;
|
| + pData->pPtr = pPtr;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + Fts5Auxdata *pData;
|
| + void *pRet = 0;
|
| +
|
| + for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){
|
| + if( pData->pAux==pCsr->pAux ) break;
|
| + }
|
| +
|
| + if( pData ){
|
| + pRet = pData->pPtr;
|
| + if( bClear ){
|
| + pData->pPtr = 0;
|
| + pData->xDelete = 0;
|
| + }
|
| + }
|
| +
|
| + return pRet;
|
| +}
|
| +
|
| +static void fts5ApiPhraseNext(
|
| + Fts5Context *pCtx,
|
| + Fts5PhraseIter *pIter,
|
| + int *piCol, int *piOff
|
| +){
|
| + if( pIter->a>=pIter->b ){
|
| + *piCol = -1;
|
| + *piOff = -1;
|
| + }else{
|
| + int iVal;
|
| + pIter->a += fts5GetVarint32(pIter->a, iVal);
|
| + if( iVal==1 ){
|
| + pIter->a += fts5GetVarint32(pIter->a, iVal);
|
| + *piCol = iVal;
|
| + *piOff = 0;
|
| + pIter->a += fts5GetVarint32(pIter->a, iVal);
|
| + }
|
| + *piOff += (iVal-2);
|
| + }
|
| +}
|
| +
|
| +static void fts5ApiPhraseFirst(
|
| + Fts5Context *pCtx,
|
| + int iPhrase,
|
| + Fts5PhraseIter *pIter,
|
| + int *piCol, int *piOff
|
| +){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a);
|
| + pIter->b = &pIter->a[n];
|
| + *piCol = 0;
|
| + *piOff = 0;
|
| + fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
|
| +}
|
| +
|
| +static int fts5ApiQueryPhrase(Fts5Context*, int, void*,
|
| + int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
|
| +);
|
| +
|
| +static const Fts5ExtensionApi sFts5Api = {
|
| + 2, /* iVersion */
|
| + fts5ApiUserData,
|
| + fts5ApiColumnCount,
|
| + fts5ApiRowCount,
|
| + fts5ApiColumnTotalSize,
|
| + fts5ApiTokenize,
|
| + fts5ApiPhraseCount,
|
| + fts5ApiPhraseSize,
|
| + fts5ApiInstCount,
|
| + fts5ApiInst,
|
| + fts5ApiRowid,
|
| + fts5ApiColumnText,
|
| + fts5ApiColumnSize,
|
| + fts5ApiQueryPhrase,
|
| + fts5ApiSetAuxdata,
|
| + fts5ApiGetAuxdata,
|
| + fts5ApiPhraseFirst,
|
| + fts5ApiPhraseNext,
|
| +};
|
| +
|
| +
|
| +/*
|
| +** Implementation of API function xQueryPhrase().
|
| +*/
|
| +static int fts5ApiQueryPhrase(
|
| + Fts5Context *pCtx,
|
| + int iPhrase,
|
| + void *pUserData,
|
| + int(*xCallback)(const Fts5ExtensionApi*, Fts5Context*, void*)
|
| +){
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
|
| + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
|
| + int rc;
|
| + Fts5Cursor *pNew = 0;
|
| +
|
| + rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew);
|
| + if( rc==SQLITE_OK ){
|
| + Fts5Config *pConf = pTab->pConfig;
|
| + pNew->ePlan = FTS5_PLAN_MATCH;
|
| + pNew->iFirstRowid = SMALLEST_INT64;
|
| + pNew->iLastRowid = LARGEST_INT64;
|
| + pNew->base.pVtab = (sqlite3_vtab*)pTab;
|
| + rc = sqlite3Fts5ExprClonePhrase(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr);
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + for(rc = fts5CursorFirst(pTab, pNew, 0);
|
| + rc==SQLITE_OK && CsrFlagTest(pNew, FTS5CSR_EOF)==0;
|
| + rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew)
|
| + ){
|
| + rc = xCallback(&sFts5Api, (Fts5Context*)pNew, pUserData);
|
| + if( rc!=SQLITE_OK ){
|
| + if( rc==SQLITE_DONE ) rc = SQLITE_OK;
|
| + break;
|
| + }
|
| + }
|
| + }
|
| +
|
| + fts5CloseMethod((sqlite3_vtab_cursor*)pNew);
|
| + return rc;
|
| +}
|
| +
|
| +static void fts5ApiInvoke(
|
| + Fts5Auxiliary *pAux,
|
| + Fts5Cursor *pCsr,
|
| + sqlite3_context *context,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| + assert( pCsr->pAux==0 );
|
| + pCsr->pAux = pAux;
|
| + pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv);
|
| + pCsr->pAux = 0;
|
| +}
|
| +
|
| +static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){
|
| + Fts5Cursor *pCsr;
|
| + for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
|
| + if( pCsr->iCsrId==iCsrId ) break;
|
| + }
|
| + return pCsr;
|
| +}
|
| +
|
| +static void fts5ApiCallback(
|
| + sqlite3_context *context,
|
| + int argc,
|
| + sqlite3_value **argv
|
| +){
|
| +
|
| + Fts5Auxiliary *pAux;
|
| + Fts5Cursor *pCsr;
|
| + i64 iCsrId;
|
| +
|
| + assert( argc>=1 );
|
| + pAux = (Fts5Auxiliary*)sqlite3_user_data(context);
|
| + iCsrId = sqlite3_value_int64(argv[0]);
|
| +
|
| + pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId);
|
| + if( pCsr==0 ){
|
| + char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId);
|
| + sqlite3_result_error(context, zErr, -1);
|
| + sqlite3_free(zErr);
|
| + }else{
|
| + fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]);
|
| + }
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Given cursor id iId, return a pointer to the corresponding Fts5Index
|
| +** object. Or NULL If the cursor id does not exist.
|
| +**
|
| +** If successful, set *ppConfig to point to the associated config object
|
| +** before returning.
|
| +*/
|
| +static Fts5Index *sqlite3Fts5IndexFromCsrid(
|
| + Fts5Global *pGlobal, /* FTS5 global context for db handle */
|
| + i64 iCsrId, /* Id of cursor to find */
|
| + Fts5Config **ppConfig /* OUT: Configuration object */
|
| +){
|
| + Fts5Cursor *pCsr;
|
| + Fts5Table *pTab;
|
| +
|
| + pCsr = fts5CursorFromCsrid(pGlobal, iCsrId);
|
| + pTab = (Fts5Table*)pCsr->base.pVtab;
|
| + *ppConfig = pTab->pConfig;
|
| +
|
| + return pTab->pIndex;
|
| +}
|
| +
|
| +/*
|
| +** Return a "position-list blob" corresponding to the current position of
|
| +** cursor pCsr via sqlite3_result_blob(). A position-list blob contains
|
| +** the current position-list for each phrase in the query associated with
|
| +** cursor pCsr.
|
| +**
|
| +** A position-list blob begins with (nPhrase-1) varints, where nPhrase is
|
| +** the number of phrases in the query. Following the varints are the
|
| +** concatenated position lists for each phrase, in order.
|
| +**
|
| +** The first varint (if it exists) contains the size of the position list
|
| +** for phrase 0. The second (same disclaimer) contains the size of position
|
| +** list 1. And so on. There is no size field for the final position list,
|
| +** as it can be derived from the total size of the blob.
|
| +*/
|
| +static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
|
| + int i;
|
| + int rc = SQLITE_OK;
|
| + int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
|
| + Fts5Buffer val;
|
| +
|
| + memset(&val, 0, sizeof(Fts5Buffer));
|
| +
|
| + /* Append the varints */
|
| + for(i=0; i<(nPhrase-1); i++){
|
| + const u8 *dummy;
|
| + int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
|
| + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
|
| + }
|
| +
|
| + /* Append the position lists */
|
| + for(i=0; i<nPhrase; i++){
|
| + const u8 *pPoslist;
|
| + int nPoslist;
|
| + nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
|
| + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
|
| + }
|
| +
|
| + sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free);
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** This is the xColumn method, called by SQLite to request a value from
|
| +** the row that the supplied cursor currently points to.
|
| +*/
|
| +static int fts5ColumnMethod(
|
| + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
|
| + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
|
| + int iCol /* Index of column to read value from */
|
| +){
|
| + Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
|
| + Fts5Config *pConfig = pTab->pConfig;
|
| + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
|
| + int rc = SQLITE_OK;
|
| +
|
| + assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
|
| +
|
| + if( pCsr->ePlan==FTS5_PLAN_SPECIAL ){
|
| + if( iCol==pConfig->nCol ){
|
| + sqlite3_result_int64(pCtx, pCsr->iSpecial);
|
| + }
|
| + }else
|
| +
|
| + if( iCol==pConfig->nCol ){
|
| + /* User is requesting the value of the special column with the same name
|
| + ** as the table. Return the cursor integer id number. This value is only
|
| + ** useful in that it may be passed as the first argument to an FTS5
|
| + ** auxiliary function. */
|
| + sqlite3_result_int64(pCtx, pCsr->iCsrId);
|
| + }else if( iCol==pConfig->nCol+1 ){
|
| +
|
| + /* The value of the "rank" column. */
|
| + if( pCsr->ePlan==FTS5_PLAN_SOURCE ){
|
| + fts5PoslistBlob(pCtx, pCsr);
|
| + }else if(
|
| + pCsr->ePlan==FTS5_PLAN_MATCH
|
| + || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH
|
| + ){
|
| + if( pCsr->pRank || SQLITE_OK==(rc = fts5FindRankFunction(pCsr)) ){
|
| + fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg);
|
| + }
|
| + }
|
| + }else if( !fts5IsContentless(pTab) ){
|
| + rc = fts5SeekCursor(pCsr, 1);
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** This routine implements the xFindFunction method for the FTS3
|
| +** virtual table.
|
| +*/
|
| +static int fts5FindFunctionMethod(
|
| + sqlite3_vtab *pVtab, /* Virtual table handle */
|
| + int nArg, /* Number of SQL function arguments */
|
| + const char *zName, /* Name of SQL function */
|
| + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
|
| + void **ppArg /* OUT: User data for *pxFunc */
|
| +){
|
| + Fts5Table *pTab = (Fts5Table*)pVtab;
|
| + Fts5Auxiliary *pAux;
|
| +
|
| + pAux = fts5FindAuxiliary(pTab, zName);
|
| + if( pAux ){
|
| + *pxFunc = fts5ApiCallback;
|
| + *ppArg = (void*)pAux;
|
| + return 1;
|
| + }
|
| +
|
| + /* No function of the specified name was found. Return 0. */
|
| + return 0;
|
| +}
|
| +
|
| +/*
|
| +** Implementation of FTS5 xRename method. Rename an fts5 table.
|
| +*/
|
| +static int fts5RenameMethod(
|
| + sqlite3_vtab *pVtab, /* Virtual table handle */
|
| + const char *zName /* New name of table */
|
| +){
|
| + Fts5Table *pTab = (Fts5Table*)pVtab;
|
| + return sqlite3Fts5StorageRename(pTab->pStorage, zName);
|
| +}
|
| +
|
| +/*
|
| +** The xSavepoint() method.
|
| +**
|
| +** Flush the contents of the pending-terms table to disk.
|
| +*/
|
| +static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
|
| + Fts5Table *pTab = (Fts5Table*)pVtab;
|
| + fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
|
| + fts5TripCursors(pTab);
|
| + return sqlite3Fts5StorageSync(pTab->pStorage, 0);
|
| +}
|
| +
|
| +/*
|
| +** The xRelease() method.
|
| +**
|
| +** This is a no-op.
|
| +*/
|
| +static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
|
| + Fts5Table *pTab = (Fts5Table*)pVtab;
|
| + fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
|
| + fts5TripCursors(pTab);
|
| + return sqlite3Fts5StorageSync(pTab->pStorage, 0);
|
| +}
|
| +
|
| +/*
|
| +** The xRollbackTo() method.
|
| +**
|
| +** Discard the contents of the pending terms table.
|
| +*/
|
| +static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
|
| + Fts5Table *pTab = (Fts5Table*)pVtab;
|
| + fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
|
| + fts5TripCursors(pTab);
|
| + return sqlite3Fts5StorageRollback(pTab->pStorage);
|
| +}
|
| +
|
| +/*
|
| +** Register a new auxiliary function with global context pGlobal.
|
| +*/
|
| +static int fts5CreateAux(
|
| + fts5_api *pApi, /* Global context (one per db handle) */
|
| + const char *zName, /* Name of new function */
|
| + void *pUserData, /* User data for aux. function */
|
| + fts5_extension_function xFunc, /* Aux. function implementation */
|
| + void(*xDestroy)(void*) /* Destructor for pUserData */
|
| +){
|
| + Fts5Global *pGlobal = (Fts5Global*)pApi;
|
| + int rc = sqlite3_overload_function(pGlobal->db, zName, -1);
|
| + if( rc==SQLITE_OK ){
|
| + Fts5Auxiliary *pAux;
|
| + int nName; /* Size of zName in bytes, including \0 */
|
| + int nByte; /* Bytes of space to allocate */
|
| +
|
| + nName = (int)strlen(zName) + 1;
|
| + nByte = sizeof(Fts5Auxiliary) + nName;
|
| + pAux = (Fts5Auxiliary*)sqlite3_malloc(nByte);
|
| + if( pAux ){
|
| + memset(pAux, 0, nByte);
|
| + pAux->zFunc = (char*)&pAux[1];
|
| + memcpy(pAux->zFunc, zName, nName);
|
| + pAux->pGlobal = pGlobal;
|
| + pAux->pUserData = pUserData;
|
| + pAux->xFunc = xFunc;
|
| + pAux->xDestroy = xDestroy;
|
| + pAux->pNext = pGlobal->pAux;
|
| + pGlobal->pAux = pAux;
|
| + }else{
|
| + rc = SQLITE_NOMEM;
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Register a new tokenizer. This is the implementation of the
|
| +** fts5_api.xCreateTokenizer() method.
|
| +*/
|
| +static int fts5CreateTokenizer(
|
| + fts5_api *pApi, /* Global context (one per db handle) */
|
| + const char *zName, /* Name of new function */
|
| + void *pUserData, /* User data for aux. function */
|
| + fts5_tokenizer *pTokenizer, /* Tokenizer implementation */
|
| + void(*xDestroy)(void*) /* Destructor for pUserData */
|
| +){
|
| + Fts5Global *pGlobal = (Fts5Global*)pApi;
|
| + Fts5TokenizerModule *pNew;
|
| + int nName; /* Size of zName and its \0 terminator */
|
| + int nByte; /* Bytes of space to allocate */
|
| + int rc = SQLITE_OK;
|
| +
|
| + nName = (int)strlen(zName) + 1;
|
| + nByte = sizeof(Fts5TokenizerModule) + nName;
|
| + pNew = (Fts5TokenizerModule*)sqlite3_malloc(nByte);
|
| + if( pNew ){
|
| + memset(pNew, 0, nByte);
|
| + pNew->zName = (char*)&pNew[1];
|
| + memcpy(pNew->zName, zName, nName);
|
| + pNew->pUserData = pUserData;
|
| + pNew->x = *pTokenizer;
|
| + pNew->xDestroy = xDestroy;
|
| + pNew->pNext = pGlobal->pTok;
|
| + pGlobal->pTok = pNew;
|
| + if( pNew->pNext==0 ){
|
| + pGlobal->pDfltTok = pNew;
|
| + }
|
| + }else{
|
| + rc = SQLITE_NOMEM;
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +static Fts5TokenizerModule *fts5LocateTokenizer(
|
| + Fts5Global *pGlobal,
|
| + const char *zName
|
| +){
|
| + Fts5TokenizerModule *pMod = 0;
|
| +
|
| + if( zName==0 ){
|
| + pMod = pGlobal->pDfltTok;
|
| + }else{
|
| + for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
|
| + if( sqlite3_stricmp(zName, pMod->zName)==0 ) break;
|
| + }
|
| + }
|
| +
|
| + return pMod;
|
| +}
|
| +
|
| +/*
|
| +** Find a tokenizer. This is the implementation of the
|
| +** fts5_api.xFindTokenizer() method.
|
| +*/
|
| +static int fts5FindTokenizer(
|
| + fts5_api *pApi, /* Global context (one per db handle) */
|
| + const char *zName, /* Name of new function */
|
| + void **ppUserData,
|
| + fts5_tokenizer *pTokenizer /* Populate this object */
|
| +){
|
| + int rc = SQLITE_OK;
|
| + Fts5TokenizerModule *pMod;
|
| +
|
| + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName);
|
| + if( pMod ){
|
| + *pTokenizer = pMod->x;
|
| + *ppUserData = pMod->pUserData;
|
| + }else{
|
| + memset(pTokenizer, 0, sizeof(fts5_tokenizer));
|
| + rc = SQLITE_ERROR;
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +static int sqlite3Fts5GetTokenizer(
|
| + Fts5Global *pGlobal,
|
| + const char **azArg,
|
| + int nArg,
|
| + Fts5Tokenizer **ppTok,
|
| + fts5_tokenizer **ppTokApi,
|
| + char **pzErr
|
| +){
|
| + Fts5TokenizerModule *pMod;
|
| + int rc = SQLITE_OK;
|
| +
|
| + pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]);
|
| + if( pMod==0 ){
|
| + assert( nArg>0 );
|
| + rc = SQLITE_ERROR;
|
| + *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
|
| + }else{
|
| + rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok);
|
| + *ppTokApi = &pMod->x;
|
| + if( rc!=SQLITE_OK && pzErr ){
|
| + *pzErr = sqlite3_mprintf("error in tokenizer constructor");
|
| + }
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + *ppTokApi = 0;
|
| + *ppTok = 0;
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +static void fts5ModuleDestroy(void *pCtx){
|
| + Fts5TokenizerModule *pTok, *pNextTok;
|
| + Fts5Auxiliary *pAux, *pNextAux;
|
| + Fts5Global *pGlobal = (Fts5Global*)pCtx;
|
| +
|
| + for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){
|
| + pNextAux = pAux->pNext;
|
| + if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData);
|
| + sqlite3_free(pAux);
|
| + }
|
| +
|
| + for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){
|
| + pNextTok = pTok->pNext;
|
| + if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData);
|
| + sqlite3_free(pTok);
|
| + }
|
| +
|
| + sqlite3_free(pGlobal);
|
| +}
|
| +
|
| +static void fts5Fts5Func(
|
| + sqlite3_context *pCtx, /* Function call context */
|
| + int nArg, /* Number of args */
|
| + sqlite3_value **apVal /* Function arguments */
|
| +){
|
| + Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
|
| + char buf[8];
|
| + assert( nArg==0 );
|
| + assert( sizeof(buf)>=sizeof(pGlobal) );
|
| + memcpy(buf, (void*)&pGlobal, sizeof(pGlobal));
|
| + sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT);
|
| +}
|
| +
|
| +/*
|
| +** Implementation of fts5_source_id() function.
|
| +*/
|
| +static void fts5SourceIdFunc(
|
| + sqlite3_context *pCtx, /* Function call context */
|
| + int nArg, /* Number of args */
|
| + sqlite3_value **apVal /* Function arguments */
|
| +){
|
| + assert( nArg==0 );
|
| + sqlite3_result_text(pCtx, "fts5: 2016-01-20 15:27:19 17efb4209f97fb4971656086b138599a91a75ff9", -1, SQLITE_TRANSIENT);
|
| +}
|
| +
|
| +static int fts5Init(sqlite3 *db){
|
| + static const sqlite3_module fts5Mod = {
|
| + /* iVersion */ 2,
|
| + /* xCreate */ fts5CreateMethod,
|
| + /* xConnect */ fts5ConnectMethod,
|
| + /* xBestIndex */ fts5BestIndexMethod,
|
| + /* xDisconnect */ fts5DisconnectMethod,
|
| + /* xDestroy */ fts5DestroyMethod,
|
| + /* xOpen */ fts5OpenMethod,
|
| + /* xClose */ fts5CloseMethod,
|
| + /* xFilter */ fts5FilterMethod,
|
| + /* xNext */ fts5NextMethod,
|
| + /* xEof */ fts5EofMethod,
|
| + /* xColumn */ fts5ColumnMethod,
|
| + /* xRowid */ fts5RowidMethod,
|
| + /* xUpdate */ fts5UpdateMethod,
|
| + /* xBegin */ fts5BeginMethod,
|
| + /* xSync */ fts5SyncMethod,
|
| + /* xCommit */ fts5CommitMethod,
|
| + /* xRollback */ fts5RollbackMethod,
|
| + /* xFindFunction */ fts5FindFunctionMethod,
|
| + /* xRename */ fts5RenameMethod,
|
| + /* xSavepoint */ fts5SavepointMethod,
|
| + /* xRelease */ fts5ReleaseMethod,
|
| + /* xRollbackTo */ fts5RollbackToMethod,
|
| + };
|
| +
|
| + int rc;
|
| + Fts5Global *pGlobal = 0;
|
| +
|
| + pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global));
|
| + if( pGlobal==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + void *p = (void*)pGlobal;
|
| + memset(pGlobal, 0, sizeof(Fts5Global));
|
| + pGlobal->db = db;
|
| + pGlobal->api.iVersion = 2;
|
| + pGlobal->api.xCreateFunction = fts5CreateAux;
|
| + pGlobal->api.xCreateTokenizer = fts5CreateTokenizer;
|
| + pGlobal->api.xFindTokenizer = fts5FindTokenizer;
|
| + rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy);
|
| + if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db);
|
| + if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db);
|
| + if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
|
| + if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
|
| + if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db);
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3_create_function(
|
| + db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
|
| + );
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3_create_function(
|
| + db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0
|
| + );
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** The following functions are used to register the module with SQLite. If
|
| +** this module is being built as part of the SQLite core (SQLITE_CORE is
|
| +** defined), then sqlite3_open() will call sqlite3Fts5Init() directly.
|
| +**
|
| +** Or, if this module is being built as a loadable extension,
|
| +** sqlite3Fts5Init() is omitted and the two standard entry points
|
| +** sqlite3_fts_init() and sqlite3_fts5_init() defined instead.
|
| +*/
|
| +#ifndef SQLITE_CORE
|
| +#ifdef _WIN32
|
| +__declspec(dllexport)
|
| +#endif
|
| +SQLITE_API int SQLITE_STDCALL sqlite3_fts_init(
|
| + sqlite3 *db,
|
| + char **pzErrMsg,
|
| + const sqlite3_api_routines *pApi
|
| +){
|
| + SQLITE_EXTENSION_INIT2(pApi);
|
| + (void)pzErrMsg; /* Unused parameter */
|
| + return fts5Init(db);
|
| +}
|
| +
|
| +#ifdef _WIN32
|
| +__declspec(dllexport)
|
| +#endif
|
| +SQLITE_API int SQLITE_STDCALL sqlite3_fts5_init(
|
| + sqlite3 *db,
|
| + char **pzErrMsg,
|
| + const sqlite3_api_routines *pApi
|
| +){
|
| + SQLITE_EXTENSION_INIT2(pApi);
|
| + (void)pzErrMsg; /* Unused parameter */
|
| + return fts5Init(db);
|
| +}
|
| +#else
|
| +SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3 *db){
|
| + return fts5Init(db);
|
| +}
|
| +#endif
|
| +
|
| +/*
|
| +** 2014 May 31
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +*/
|
| +
|
| +
|
| +
|
| +/* #include "fts5Int.h" */
|
| +
|
| +struct Fts5Storage {
|
| + Fts5Config *pConfig;
|
| + Fts5Index *pIndex;
|
| + int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */
|
| + i64 nTotalRow; /* Total number of rows in FTS table */
|
| + i64 *aTotalSize; /* Total sizes of each column */
|
| + sqlite3_stmt *aStmt[11];
|
| +};
|
| +
|
| +
|
| +#if FTS5_STMT_SCAN_ASC!=0
|
| +# error "FTS5_STMT_SCAN_ASC mismatch"
|
| +#endif
|
| +#if FTS5_STMT_SCAN_DESC!=1
|
| +# error "FTS5_STMT_SCAN_DESC mismatch"
|
| +#endif
|
| +#if FTS5_STMT_LOOKUP!=2
|
| +# error "FTS5_STMT_LOOKUP mismatch"
|
| +#endif
|
| +
|
| +#define FTS5_STMT_INSERT_CONTENT 3
|
| +#define FTS5_STMT_REPLACE_CONTENT 4
|
| +#define FTS5_STMT_DELETE_CONTENT 5
|
| +#define FTS5_STMT_REPLACE_DOCSIZE 6
|
| +#define FTS5_STMT_DELETE_DOCSIZE 7
|
| +#define FTS5_STMT_LOOKUP_DOCSIZE 8
|
| +#define FTS5_STMT_REPLACE_CONFIG 9
|
| +#define FTS5_STMT_SCAN 10
|
| +
|
| +/*
|
| +** Prepare the two insert statements - Fts5Storage.pInsertContent and
|
| +** Fts5Storage.pInsertDocsize - if they have not already been prepared.
|
| +** Return SQLITE_OK if successful, or an SQLite error code if an error
|
| +** occurs.
|
| +*/
|
| +static int fts5StorageGetStmt(
|
| + Fts5Storage *p, /* Storage handle */
|
| + int eStmt, /* FTS5_STMT_XXX constant */
|
| + sqlite3_stmt **ppStmt, /* OUT: Prepared statement handle */
|
| + char **pzErrMsg /* OUT: Error message (if any) */
|
| +){
|
| + int rc = SQLITE_OK;
|
| +
|
| + /* If there is no %_docsize table, there should be no requests for
|
| + ** statements to operate on it. */
|
| + assert( p->pConfig->bColumnsize || (
|
| + eStmt!=FTS5_STMT_REPLACE_DOCSIZE
|
| + && eStmt!=FTS5_STMT_DELETE_DOCSIZE
|
| + && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE
|
| + ));
|
| +
|
| + assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
|
| + if( p->aStmt[eStmt]==0 ){
|
| + const char *azStmt[] = {
|
| + "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
|
| + "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
|
| + "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */
|
| +
|
| + "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */
|
| + "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */
|
| + "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */
|
| + "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */
|
| + "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */
|
| +
|
| + "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */
|
| +
|
| + "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */
|
| + "SELECT %s FROM %s AS T", /* SCAN */
|
| + };
|
| + Fts5Config *pC = p->pConfig;
|
| + char *zSql = 0;
|
| +
|
| + switch( eStmt ){
|
| + case FTS5_STMT_SCAN:
|
| + zSql = sqlite3_mprintf(azStmt[eStmt],
|
| + pC->zContentExprlist, pC->zContent
|
| + );
|
| + break;
|
| +
|
| + case FTS5_STMT_SCAN_ASC:
|
| + case FTS5_STMT_SCAN_DESC:
|
| + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist,
|
| + pC->zContent, pC->zContentRowid, pC->zContentRowid,
|
| + pC->zContentRowid
|
| + );
|
| + break;
|
| +
|
| + case FTS5_STMT_LOOKUP:
|
| + zSql = sqlite3_mprintf(azStmt[eStmt],
|
| + pC->zContentExprlist, pC->zContent, pC->zContentRowid
|
| + );
|
| + break;
|
| +
|
| + case FTS5_STMT_INSERT_CONTENT:
|
| + case FTS5_STMT_REPLACE_CONTENT: {
|
| + int nCol = pC->nCol + 1;
|
| + char *zBind;
|
| + int i;
|
| +
|
| + zBind = sqlite3_malloc(1 + nCol*2);
|
| + if( zBind ){
|
| + for(i=0; i<nCol; i++){
|
| + zBind[i*2] = '?';
|
| + zBind[i*2 + 1] = ',';
|
| + }
|
| + zBind[i*2-1] = '\0';
|
| + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
|
| + sqlite3_free(zBind);
|
| + }
|
| + break;
|
| + }
|
| +
|
| + default:
|
| + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
|
| + break;
|
| + }
|
| +
|
| + if( zSql==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
|
| + sqlite3_free(zSql);
|
| + if( rc!=SQLITE_OK && pzErrMsg ){
|
| + *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
|
| + }
|
| + }
|
| + }
|
| +
|
| + *ppStmt = p->aStmt[eStmt];
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +static int fts5ExecPrintf(
|
| + sqlite3 *db,
|
| + char **pzErr,
|
| + const char *zFormat,
|
| + ...
|
| +){
|
| + int rc;
|
| + va_list ap; /* ... printf arguments */
|
| + char *zSql;
|
| +
|
| + va_start(ap, zFormat);
|
| + zSql = sqlite3_vmprintf(zFormat, ap);
|
| +
|
| + if( zSql==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + rc = sqlite3_exec(db, zSql, 0, 0, pzErr);
|
| + sqlite3_free(zSql);
|
| + }
|
| +
|
| + va_end(ap);
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error
|
| +** code otherwise.
|
| +*/
|
| +static int sqlite3Fts5DropAll(Fts5Config *pConfig){
|
| + int rc = fts5ExecPrintf(pConfig->db, 0,
|
| + "DROP TABLE IF EXISTS %Q.'%q_data';"
|
| + "DROP TABLE IF EXISTS %Q.'%q_idx';"
|
| + "DROP TABLE IF EXISTS %Q.'%q_config';",
|
| + pConfig->zDb, pConfig->zName,
|
| + pConfig->zDb, pConfig->zName,
|
| + pConfig->zDb, pConfig->zName
|
| + );
|
| + if( rc==SQLITE_OK && pConfig->bColumnsize ){
|
| + rc = fts5ExecPrintf(pConfig->db, 0,
|
| + "DROP TABLE IF EXISTS %Q.'%q_docsize';",
|
| + pConfig->zDb, pConfig->zName
|
| + );
|
| + }
|
| + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
|
| + rc = fts5ExecPrintf(pConfig->db, 0,
|
| + "DROP TABLE IF EXISTS %Q.'%q_content';",
|
| + pConfig->zDb, pConfig->zName
|
| + );
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static void fts5StorageRenameOne(
|
| + Fts5Config *pConfig, /* Current FTS5 configuration */
|
| + int *pRc, /* IN/OUT: Error code */
|
| + const char *zTail, /* Tail of table name e.g. "data", "config" */
|
| + const char *zName /* New name of FTS5 table */
|
| +){
|
| + if( *pRc==SQLITE_OK ){
|
| + *pRc = fts5ExecPrintf(pConfig->db, 0,
|
| + "ALTER TABLE %Q.'%q_%s' RENAME TO '%q_%s';",
|
| + pConfig->zDb, pConfig->zName, zTail, zName, zTail
|
| + );
|
| + }
|
| +}
|
| +
|
| +static int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){
|
| + Fts5Config *pConfig = pStorage->pConfig;
|
| + int rc = sqlite3Fts5StorageSync(pStorage, 1);
|
| +
|
| + fts5StorageRenameOne(pConfig, &rc, "data", zName);
|
| + fts5StorageRenameOne(pConfig, &rc, "idx", zName);
|
| + fts5StorageRenameOne(pConfig, &rc, "config", zName);
|
| + if( pConfig->bColumnsize ){
|
| + fts5StorageRenameOne(pConfig, &rc, "docsize", zName);
|
| + }
|
| + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
|
| + fts5StorageRenameOne(pConfig, &rc, "content", zName);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Create the shadow table named zPost, with definition zDefn. Return
|
| +** SQLITE_OK if successful, or an SQLite error code otherwise.
|
| +*/
|
| +static int sqlite3Fts5CreateTable(
|
| + Fts5Config *pConfig, /* FTS5 configuration */
|
| + const char *zPost, /* Shadow table to create (e.g. "content") */
|
| + const char *zDefn, /* Columns etc. for shadow table */
|
| + int bWithout, /* True for without rowid */
|
| + char **pzErr /* OUT: Error message */
|
| +){
|
| + int rc;
|
| + char *zErr = 0;
|
| +
|
| + rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s",
|
| + pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":""
|
| + );
|
| + if( zErr ){
|
| + *pzErr = sqlite3_mprintf(
|
| + "fts5: error creating shadow table %q_%s: %s",
|
| + pConfig->zName, zPost, zErr
|
| + );
|
| + sqlite3_free(zErr);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Open a new Fts5Index handle. If the bCreate argument is true, create
|
| +** and initialize the underlying tables
|
| +**
|
| +** If successful, set *pp to point to the new object and return SQLITE_OK.
|
| +** Otherwise, set *pp to NULL and return an SQLite error code.
|
| +*/
|
| +static int sqlite3Fts5StorageOpen(
|
| + Fts5Config *pConfig,
|
| + Fts5Index *pIndex,
|
| + int bCreate,
|
| + Fts5Storage **pp,
|
| + char **pzErr /* OUT: Error message */
|
| +){
|
| + int rc = SQLITE_OK;
|
| + Fts5Storage *p; /* New object */
|
| + int nByte; /* Bytes of space to allocate */
|
| +
|
| + nByte = sizeof(Fts5Storage) /* Fts5Storage object */
|
| + + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */
|
| + *pp = p = (Fts5Storage*)sqlite3_malloc(nByte);
|
| + if( !p ) return SQLITE_NOMEM;
|
| +
|
| + memset(p, 0, nByte);
|
| + p->aTotalSize = (i64*)&p[1];
|
| + p->pConfig = pConfig;
|
| + p->pIndex = pIndex;
|
| +
|
| + if( bCreate ){
|
| + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
|
| + int nDefn = 32 + pConfig->nCol*10;
|
| + char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
|
| + if( zDefn==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + int i;
|
| + int iOff;
|
| + sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY");
|
| + iOff = (int)strlen(zDefn);
|
| + for(i=0; i<pConfig->nCol; i++){
|
| + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
|
| + iOff += (int)strlen(&zDefn[iOff]);
|
| + }
|
| + rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
|
| + }
|
| + sqlite3_free(zDefn);
|
| + }
|
| +
|
| + if( rc==SQLITE_OK && pConfig->bColumnsize ){
|
| + rc = sqlite3Fts5CreateTable(
|
| + pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
|
| + );
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5CreateTable(
|
| + pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
|
| + );
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION);
|
| + }
|
| + }
|
| +
|
| + if( rc ){
|
| + sqlite3Fts5StorageClose(p);
|
| + *pp = 0;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen().
|
| +*/
|
| +static int sqlite3Fts5StorageClose(Fts5Storage *p){
|
| + int rc = SQLITE_OK;
|
| + if( p ){
|
| + int i;
|
| +
|
| + /* Finalize all SQL statements */
|
| + for(i=0; i<(int)ArraySize(p->aStmt); i++){
|
| + sqlite3_finalize(p->aStmt[i]);
|
| + }
|
| +
|
| + sqlite3_free(p);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +typedef struct Fts5InsertCtx Fts5InsertCtx;
|
| +struct Fts5InsertCtx {
|
| + Fts5Storage *pStorage;
|
| + int iCol;
|
| + int szCol; /* Size of column value in tokens */
|
| +};
|
| +
|
| +/*
|
| +** Tokenization callback used when inserting tokens into the FTS index.
|
| +*/
|
| +static int fts5StorageInsertCallback(
|
| + void *pContext, /* Pointer to Fts5InsertCtx object */
|
| + int tflags,
|
| + const char *pToken, /* Buffer containing token */
|
| + int nToken, /* Size of token in bytes */
|
| + int iStart, /* Start offset of token */
|
| + int iEnd /* End offset of token */
|
| +){
|
| + Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
|
| + Fts5Index *pIdx = pCtx->pStorage->pIndex;
|
| + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
|
| + pCtx->szCol++;
|
| + }
|
| + return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
|
| +}
|
| +
|
| +/*
|
| +** If a row with rowid iDel is present in the %_content table, add the
|
| +** delete-markers to the FTS index necessary to delete it. Do not actually
|
| +** remove the %_content row at this time though.
|
| +*/
|
| +static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + sqlite3_stmt *pSeek; /* SELECT to read row iDel from %_data */
|
| + int rc; /* Return code */
|
| +
|
| + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
|
| + if( rc==SQLITE_OK ){
|
| + int rc2;
|
| + sqlite3_bind_int64(pSeek, 1, iDel);
|
| + if( sqlite3_step(pSeek)==SQLITE_ROW ){
|
| + int iCol;
|
| + Fts5InsertCtx ctx;
|
| + ctx.pStorage = p;
|
| + ctx.iCol = -1;
|
| + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
|
| + for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
|
| + if( pConfig->abUnindexed[iCol-1] ) continue;
|
| + ctx.szCol = 0;
|
| + rc = sqlite3Fts5Tokenize(pConfig,
|
| + FTS5_TOKENIZE_DOCUMENT,
|
| + (const char*)sqlite3_column_text(pSeek, iCol),
|
| + sqlite3_column_bytes(pSeek, iCol),
|
| + (void*)&ctx,
|
| + fts5StorageInsertCallback
|
| + );
|
| + p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
|
| + }
|
| + p->nTotalRow--;
|
| + }
|
| + rc2 = sqlite3_reset(pSeek);
|
| + if( rc==SQLITE_OK ) rc = rc2;
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Insert a record into the %_docsize table. Specifically, do:
|
| +**
|
| +** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
|
| +**
|
| +** If there is no %_docsize table (as happens if the columnsize=0 option
|
| +** is specified when the FTS5 table is created), this function is a no-op.
|
| +*/
|
| +static int fts5StorageInsertDocsize(
|
| + Fts5Storage *p, /* Storage module to write to */
|
| + i64 iRowid, /* id value */
|
| + Fts5Buffer *pBuf /* sz value */
|
| +){
|
| + int rc = SQLITE_OK;
|
| + if( p->pConfig->bColumnsize ){
|
| + sqlite3_stmt *pReplace = 0;
|
| + rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_bind_int64(pReplace, 1, iRowid);
|
| + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
|
| + sqlite3_step(pReplace);
|
| + rc = sqlite3_reset(pReplace);
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Load the contents of the "averages" record from disk into the
|
| +** p->nTotalRow and p->aTotalSize[] variables. If successful, and if
|
| +** argument bCache is true, set the p->bTotalsValid flag to indicate
|
| +** that the contents of aTotalSize[] and nTotalRow are valid until
|
| +** further notice.
|
| +**
|
| +** Return SQLITE_OK if successful, or an SQLite error code if an error
|
| +** occurs.
|
| +*/
|
| +static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){
|
| + int rc = SQLITE_OK;
|
| + if( p->bTotalsValid==0 ){
|
| + rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize);
|
| + p->bTotalsValid = bCache;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Store the current contents of the p->nTotalRow and p->aTotalSize[]
|
| +** variables in the "averages" record on disk.
|
| +**
|
| +** Return SQLITE_OK if successful, or an SQLite error code if an error
|
| +** occurs.
|
| +*/
|
| +static int fts5StorageSaveTotals(Fts5Storage *p){
|
| + int nCol = p->pConfig->nCol;
|
| + int i;
|
| + Fts5Buffer buf;
|
| + int rc = SQLITE_OK;
|
| + memset(&buf, 0, sizeof(buf));
|
| +
|
| + sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow);
|
| + for(i=0; i<nCol; i++){
|
| + sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n);
|
| + }
|
| + sqlite3_free(buf.p);
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Remove a row from the FTS table.
|
| +*/
|
| +static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + int rc;
|
| + sqlite3_stmt *pDel = 0;
|
| +
|
| + rc = fts5StorageLoadTotals(p, 1);
|
| +
|
| + /* Delete the index records */
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5StorageDeleteFromIndex(p, iDel);
|
| + }
|
| +
|
| + /* Delete the %_docsize record */
|
| + if( rc==SQLITE_OK && pConfig->bColumnsize ){
|
| + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_bind_int64(pDel, 1, iDel);
|
| + sqlite3_step(pDel);
|
| + rc = sqlite3_reset(pDel);
|
| + }
|
| + }
|
| +
|
| + /* Delete the %_content record */
|
| + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_bind_int64(pDel, 1, iDel);
|
| + sqlite3_step(pDel);
|
| + rc = sqlite3_reset(pDel);
|
| + }
|
| + }
|
| +
|
| + /* Write the averages record */
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5StorageSaveTotals(p);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +static int sqlite3Fts5StorageSpecialDelete(
|
| + Fts5Storage *p,
|
| + i64 iDel,
|
| + sqlite3_value **apVal
|
| +){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + int rc;
|
| + sqlite3_stmt *pDel = 0;
|
| +
|
| + assert( pConfig->eContent!=FTS5_CONTENT_NORMAL );
|
| + rc = fts5StorageLoadTotals(p, 1);
|
| +
|
| + /* Delete the index records */
|
| + if( rc==SQLITE_OK ){
|
| + int iCol;
|
| + Fts5InsertCtx ctx;
|
| + ctx.pStorage = p;
|
| + ctx.iCol = -1;
|
| +
|
| + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
|
| + for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
|
| + if( pConfig->abUnindexed[iCol] ) continue;
|
| + ctx.szCol = 0;
|
| + rc = sqlite3Fts5Tokenize(pConfig,
|
| + FTS5_TOKENIZE_DOCUMENT,
|
| + (const char*)sqlite3_value_text(apVal[iCol]),
|
| + sqlite3_value_bytes(apVal[iCol]),
|
| + (void*)&ctx,
|
| + fts5StorageInsertCallback
|
| + );
|
| + p->aTotalSize[iCol] -= (i64)ctx.szCol;
|
| + }
|
| + p->nTotalRow--;
|
| + }
|
| +
|
| + /* Delete the %_docsize record */
|
| + if( pConfig->bColumnsize ){
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_bind_int64(pDel, 1, iDel);
|
| + sqlite3_step(pDel);
|
| + rc = sqlite3_reset(pDel);
|
| + }
|
| + }
|
| +
|
| + /* Write the averages record */
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5StorageSaveTotals(p);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Delete all entries in the FTS5 index.
|
| +*/
|
| +static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + int rc;
|
| +
|
| + /* Delete the contents of the %_data and %_docsize tables. */
|
| + rc = fts5ExecPrintf(pConfig->db, 0,
|
| + "DELETE FROM %Q.'%q_data';"
|
| + "DELETE FROM %Q.'%q_idx';",
|
| + pConfig->zDb, pConfig->zName,
|
| + pConfig->zDb, pConfig->zName
|
| + );
|
| + if( rc==SQLITE_OK && pConfig->bColumnsize ){
|
| + rc = fts5ExecPrintf(pConfig->db, 0,
|
| + "DELETE FROM %Q.'%q_docsize';",
|
| + pConfig->zDb, pConfig->zName
|
| + );
|
| + }
|
| +
|
| + /* Reinitialize the %_data table. This call creates the initial structure
|
| + ** and averages records. */
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5IndexReinit(p->pIndex);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static int sqlite3Fts5StorageRebuild(Fts5Storage *p){
|
| + Fts5Buffer buf = {0,0,0};
|
| + Fts5Config *pConfig = p->pConfig;
|
| + sqlite3_stmt *pScan = 0;
|
| + Fts5InsertCtx ctx;
|
| + int rc;
|
| +
|
| + memset(&ctx, 0, sizeof(Fts5InsertCtx));
|
| + ctx.pStorage = p;
|
| + rc = sqlite3Fts5StorageDeleteAll(p);
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5StorageLoadTotals(p, 1);
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
|
| + }
|
| +
|
| + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
|
| + i64 iRowid = sqlite3_column_int64(pScan, 0);
|
| +
|
| + sqlite3Fts5BufferZero(&buf);
|
| + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
|
| + for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
|
| + ctx.szCol = 0;
|
| + if( pConfig->abUnindexed[ctx.iCol]==0 ){
|
| + rc = sqlite3Fts5Tokenize(pConfig,
|
| + FTS5_TOKENIZE_DOCUMENT,
|
| + (const char*)sqlite3_column_text(pScan, ctx.iCol+1),
|
| + sqlite3_column_bytes(pScan, ctx.iCol+1),
|
| + (void*)&ctx,
|
| + fts5StorageInsertCallback
|
| + );
|
| + }
|
| + sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
|
| + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
|
| + }
|
| + p->nTotalRow++;
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5StorageInsertDocsize(p, iRowid, &buf);
|
| + }
|
| + }
|
| + sqlite3_free(buf.p);
|
| +
|
| + /* Write the averages record */
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5StorageSaveTotals(p);
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static int sqlite3Fts5StorageOptimize(Fts5Storage *p){
|
| + return sqlite3Fts5IndexOptimize(p->pIndex);
|
| +}
|
| +
|
| +static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
|
| + return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
|
| +}
|
| +
|
| +/*
|
| +** Allocate a new rowid. This is used for "external content" tables when
|
| +** a NULL value is inserted into the rowid column. The new rowid is allocated
|
| +** by inserting a dummy row into the %_docsize table. The dummy will be
|
| +** overwritten later.
|
| +**
|
| +** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In
|
| +** this case the user is required to provide a rowid explicitly.
|
| +*/
|
| +static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
|
| + int rc = SQLITE_MISMATCH;
|
| + if( p->pConfig->bColumnsize ){
|
| + sqlite3_stmt *pReplace = 0;
|
| + rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_bind_null(pReplace, 1);
|
| + sqlite3_bind_null(pReplace, 2);
|
| + sqlite3_step(pReplace);
|
| + rc = sqlite3_reset(pReplace);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Insert a new row into the FTS content table.
|
| +*/
|
| +static int sqlite3Fts5StorageContentInsert(
|
| + Fts5Storage *p,
|
| + sqlite3_value **apVal,
|
| + i64 *piRowid
|
| +){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + int rc = SQLITE_OK;
|
| +
|
| + /* Insert the new row into the %_content table. */
|
| + if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
|
| + if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
|
| + *piRowid = sqlite3_value_int64(apVal[1]);
|
| + }else{
|
| + rc = fts5StorageNewRowid(p, piRowid);
|
| + }
|
| + }else{
|
| + sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */
|
| + int i; /* Counter variable */
|
| + rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
|
| + for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
|
| + rc = sqlite3_bind_value(pInsert, i, apVal[i]);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_step(pInsert);
|
| + rc = sqlite3_reset(pInsert);
|
| + }
|
| + *piRowid = sqlite3_last_insert_rowid(pConfig->db);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Insert new entries into the FTS index and %_docsize table.
|
| +*/
|
| +static int sqlite3Fts5StorageIndexInsert(
|
| + Fts5Storage *p,
|
| + sqlite3_value **apVal,
|
| + i64 iRowid
|
| +){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + int rc = SQLITE_OK; /* Return code */
|
| + Fts5InsertCtx ctx; /* Tokenization callback context object */
|
| + Fts5Buffer buf; /* Buffer used to build up %_docsize blob */
|
| +
|
| + memset(&buf, 0, sizeof(Fts5Buffer));
|
| + ctx.pStorage = p;
|
| + rc = fts5StorageLoadTotals(p, 1);
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
|
| + }
|
| + for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
|
| + ctx.szCol = 0;
|
| + if( pConfig->abUnindexed[ctx.iCol]==0 ){
|
| + rc = sqlite3Fts5Tokenize(pConfig,
|
| + FTS5_TOKENIZE_DOCUMENT,
|
| + (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
|
| + sqlite3_value_bytes(apVal[ctx.iCol+2]),
|
| + (void*)&ctx,
|
| + fts5StorageInsertCallback
|
| + );
|
| + }
|
| + sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
|
| + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
|
| + }
|
| + p->nTotalRow++;
|
| +
|
| + /* Write the %_docsize record */
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5StorageInsertDocsize(p, iRowid, &buf);
|
| + }
|
| + sqlite3_free(buf.p);
|
| +
|
| + /* Write the averages record */
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5StorageSaveTotals(p);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + char *zSql;
|
| + int rc;
|
| +
|
| + zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'",
|
| + pConfig->zDb, pConfig->zName, zSuffix
|
| + );
|
| + if( zSql==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + sqlite3_stmt *pCnt = 0;
|
| + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0);
|
| + if( rc==SQLITE_OK ){
|
| + if( SQLITE_ROW==sqlite3_step(pCnt) ){
|
| + *pnRow = sqlite3_column_int64(pCnt, 0);
|
| + }
|
| + rc = sqlite3_finalize(pCnt);
|
| + }
|
| + }
|
| +
|
| + sqlite3_free(zSql);
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Context object used by sqlite3Fts5StorageIntegrity().
|
| +*/
|
| +typedef struct Fts5IntegrityCtx Fts5IntegrityCtx;
|
| +struct Fts5IntegrityCtx {
|
| + i64 iRowid;
|
| + int iCol;
|
| + int szCol;
|
| + u64 cksum;
|
| + Fts5Config *pConfig;
|
| +};
|
| +
|
| +/*
|
| +** Tokenization callback used by integrity check.
|
| +*/
|
| +static int fts5StorageIntegrityCallback(
|
| + void *pContext, /* Pointer to Fts5InsertCtx object */
|
| + int tflags,
|
| + const char *pToken, /* Buffer containing token */
|
| + int nToken, /* Size of token in bytes */
|
| + int iStart, /* Start offset of token */
|
| + int iEnd /* End offset of token */
|
| +){
|
| + Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
|
| + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
|
| + pCtx->szCol++;
|
| + }
|
| + pCtx->cksum ^= sqlite3Fts5IndexCksum(
|
| + pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken
|
| + );
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Check that the contents of the FTS index match that of the %_content
|
| +** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return
|
| +** some other SQLite error code if an error occurs while attempting to
|
| +** determine this.
|
| +*/
|
| +static int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
|
| + Fts5Config *pConfig = p->pConfig;
|
| + int rc; /* Return code */
|
| + int *aColSize; /* Array of size pConfig->nCol */
|
| + i64 *aTotalSize; /* Array of size pConfig->nCol */
|
| + Fts5IntegrityCtx ctx;
|
| + sqlite3_stmt *pScan;
|
| +
|
| + memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
|
| + ctx.pConfig = p->pConfig;
|
| + aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64)));
|
| + if( !aTotalSize ) return SQLITE_NOMEM;
|
| + aColSize = (int*)&aTotalSize[pConfig->nCol];
|
| + memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
|
| +
|
| + /* Generate the expected index checksum based on the contents of the
|
| + ** %_content table. This block stores the checksum in ctx.cksum. */
|
| + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
|
| + if( rc==SQLITE_OK ){
|
| + int rc2;
|
| + while( SQLITE_ROW==sqlite3_step(pScan) ){
|
| + int i;
|
| + ctx.iRowid = sqlite3_column_int64(pScan, 0);
|
| + ctx.szCol = 0;
|
| + if( pConfig->bColumnsize ){
|
| + rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
|
| + }
|
| + for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
|
| + if( pConfig->abUnindexed[i] ) continue;
|
| + ctx.iCol = i;
|
| + ctx.szCol = 0;
|
| + rc = sqlite3Fts5Tokenize(pConfig,
|
| + FTS5_TOKENIZE_DOCUMENT,
|
| + (const char*)sqlite3_column_text(pScan, i+1),
|
| + sqlite3_column_bytes(pScan, i+1),
|
| + (void*)&ctx,
|
| + fts5StorageIntegrityCallback
|
| + );
|
| + if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
|
| + rc = FTS5_CORRUPT;
|
| + }
|
| + aTotalSize[i] += ctx.szCol;
|
| + }
|
| + if( rc!=SQLITE_OK ) break;
|
| + }
|
| + rc2 = sqlite3_reset(pScan);
|
| + if( rc==SQLITE_OK ) rc = rc2;
|
| + }
|
| +
|
| + /* Test that the "totals" (sometimes called "averages") record looks Ok */
|
| + if( rc==SQLITE_OK ){
|
| + int i;
|
| + rc = fts5StorageLoadTotals(p, 0);
|
| + for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
|
| + if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT;
|
| + }
|
| + }
|
| +
|
| + /* Check that the %_docsize and %_content tables contain the expected
|
| + ** number of rows. */
|
| + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
|
| + i64 nRow = 0;
|
| + rc = fts5StorageCount(p, "content", &nRow);
|
| + if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
|
| + }
|
| + if( rc==SQLITE_OK && pConfig->bColumnsize ){
|
| + i64 nRow = 0;
|
| + rc = fts5StorageCount(p, "docsize", &nRow);
|
| + if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
|
| + }
|
| +
|
| + /* Pass the expected checksum down to the FTS index module. It will
|
| + ** verify, amongst other things, that it matches the checksum generated by
|
| + ** inspecting the index itself. */
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum);
|
| + }
|
| +
|
| + sqlite3_free(aTotalSize);
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Obtain an SQLite statement handle that may be used to read data from the
|
| +** %_content table.
|
| +*/
|
| +static int sqlite3Fts5StorageStmt(
|
| + Fts5Storage *p,
|
| + int eStmt,
|
| + sqlite3_stmt **pp,
|
| + char **pzErrMsg
|
| +){
|
| + int rc;
|
| + assert( eStmt==FTS5_STMT_SCAN_ASC
|
| + || eStmt==FTS5_STMT_SCAN_DESC
|
| + || eStmt==FTS5_STMT_LOOKUP
|
| + );
|
| + rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg);
|
| + if( rc==SQLITE_OK ){
|
| + assert( p->aStmt[eStmt]==*pp );
|
| + p->aStmt[eStmt] = 0;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Release an SQLite statement handle obtained via an earlier call to
|
| +** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function
|
| +** must match that passed to the sqlite3Fts5StorageStmt() call.
|
| +*/
|
| +static void sqlite3Fts5StorageStmtRelease(
|
| + Fts5Storage *p,
|
| + int eStmt,
|
| + sqlite3_stmt *pStmt
|
| +){
|
| + assert( eStmt==FTS5_STMT_SCAN_ASC
|
| + || eStmt==FTS5_STMT_SCAN_DESC
|
| + || eStmt==FTS5_STMT_LOOKUP
|
| + );
|
| + if( p->aStmt[eStmt]==0 ){
|
| + sqlite3_reset(pStmt);
|
| + p->aStmt[eStmt] = pStmt;
|
| + }else{
|
| + sqlite3_finalize(pStmt);
|
| + }
|
| +}
|
| +
|
| +static int fts5StorageDecodeSizeArray(
|
| + int *aCol, int nCol, /* Array to populate */
|
| + const u8 *aBlob, int nBlob /* Record to read varints from */
|
| +){
|
| + int i;
|
| + int iOff = 0;
|
| + for(i=0; i<nCol; i++){
|
| + if( iOff>=nBlob ) return 1;
|
| + iOff += fts5GetVarint32(&aBlob[iOff], aCol[i]);
|
| + }
|
| + return (iOff!=nBlob);
|
| +}
|
| +
|
| +/*
|
| +** Argument aCol points to an array of integers containing one entry for
|
| +** each table column. This function reads the %_docsize record for the
|
| +** specified rowid and populates aCol[] with the results.
|
| +**
|
| +** An SQLite error code is returned if an error occurs, or SQLITE_OK
|
| +** otherwise.
|
| +*/
|
| +static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
|
| + int nCol = p->pConfig->nCol; /* Number of user columns in table */
|
| + sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */
|
| + int rc; /* Return Code */
|
| +
|
| + assert( p->pConfig->bColumnsize );
|
| + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
|
| + if( rc==SQLITE_OK ){
|
| + int bCorrupt = 1;
|
| + sqlite3_bind_int64(pLookup, 1, iRowid);
|
| + if( SQLITE_ROW==sqlite3_step(pLookup) ){
|
| + const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
|
| + int nBlob = sqlite3_column_bytes(pLookup, 0);
|
| + if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
|
| + bCorrupt = 0;
|
| + }
|
| + }
|
| + rc = sqlite3_reset(pLookup);
|
| + if( bCorrupt && rc==SQLITE_OK ){
|
| + rc = FTS5_CORRUPT;
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
|
| + int rc = fts5StorageLoadTotals(p, 0);
|
| + if( rc==SQLITE_OK ){
|
| + *pnToken = 0;
|
| + if( iCol<0 ){
|
| + int i;
|
| + for(i=0; i<p->pConfig->nCol; i++){
|
| + *pnToken += p->aTotalSize[i];
|
| + }
|
| + }else if( iCol<p->pConfig->nCol ){
|
| + *pnToken = p->aTotalSize[iCol];
|
| + }else{
|
| + rc = SQLITE_RANGE;
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
|
| + int rc = fts5StorageLoadTotals(p, 0);
|
| + if( rc==SQLITE_OK ){
|
| + *pnRow = p->nTotalRow;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Flush any data currently held in-memory to disk.
|
| +*/
|
| +static int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){
|
| + if( bCommit && p->bTotalsValid ){
|
| + int rc = fts5StorageSaveTotals(p);
|
| + p->bTotalsValid = 0;
|
| + if( rc!=SQLITE_OK ) return rc;
|
| + }
|
| + return sqlite3Fts5IndexSync(p->pIndex, bCommit);
|
| +}
|
| +
|
| +static int sqlite3Fts5StorageRollback(Fts5Storage *p){
|
| + p->bTotalsValid = 0;
|
| + return sqlite3Fts5IndexRollback(p->pIndex);
|
| +}
|
| +
|
| +static int sqlite3Fts5StorageConfigValue(
|
| + Fts5Storage *p,
|
| + const char *z,
|
| + sqlite3_value *pVal,
|
| + int iVal
|
| +){
|
| + sqlite3_stmt *pReplace = 0;
|
| + int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0);
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC);
|
| + if( pVal ){
|
| + sqlite3_bind_value(pReplace, 2, pVal);
|
| + }else{
|
| + sqlite3_bind_int(pReplace, 2, iVal);
|
| + }
|
| + sqlite3_step(pReplace);
|
| + rc = sqlite3_reset(pReplace);
|
| + }
|
| + if( rc==SQLITE_OK && pVal ){
|
| + int iNew = p->pConfig->iCookie + 1;
|
| + rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
|
| + if( rc==SQLITE_OK ){
|
| + p->pConfig->iCookie = iNew;
|
| + }
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +
|
| +/*
|
| +** 2014 May 31
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +*/
|
| +
|
| +
|
| +/* #include "fts5Int.h" */
|
| +
|
| +/**************************************************************************
|
| +** Start of ascii tokenizer implementation.
|
| +*/
|
| +
|
| +/*
|
| +** For tokenizers with no "unicode" modifier, the set of token characters
|
| +** is the same as the set of ASCII range alphanumeric characters.
|
| +*/
|
| +static unsigned char aAsciiTokenChar[128] = {
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */
|
| + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */
|
| + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */
|
| + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */
|
| + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */
|
| + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */
|
| + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */
|
| +};
|
| +
|
| +typedef struct AsciiTokenizer AsciiTokenizer;
|
| +struct AsciiTokenizer {
|
| + unsigned char aTokenChar[128];
|
| +};
|
| +
|
| +static void fts5AsciiAddExceptions(
|
| + AsciiTokenizer *p,
|
| + const char *zArg,
|
| + int bTokenChars
|
| +){
|
| + int i;
|
| + for(i=0; zArg[i]; i++){
|
| + if( (zArg[i] & 0x80)==0 ){
|
| + p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars;
|
| + }
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Delete a "ascii" tokenizer.
|
| +*/
|
| +static void fts5AsciiDelete(Fts5Tokenizer *p){
|
| + sqlite3_free(p);
|
| +}
|
| +
|
| +/*
|
| +** Create an "ascii" tokenizer.
|
| +*/
|
| +static int fts5AsciiCreate(
|
| + void *pCtx,
|
| + const char **azArg, int nArg,
|
| + Fts5Tokenizer **ppOut
|
| +){
|
| + int rc = SQLITE_OK;
|
| + AsciiTokenizer *p = 0;
|
| + if( nArg%2 ){
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + p = sqlite3_malloc(sizeof(AsciiTokenizer));
|
| + if( p==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + int i;
|
| + memset(p, 0, sizeof(AsciiTokenizer));
|
| + memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
|
| + for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
|
| + const char *zArg = azArg[i+1];
|
| + if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
|
| + fts5AsciiAddExceptions(p, zArg, 1);
|
| + }else
|
| + if( 0==sqlite3_stricmp(azArg[i], "separators") ){
|
| + fts5AsciiAddExceptions(p, zArg, 0);
|
| + }else{
|
| + rc = SQLITE_ERROR;
|
| + }
|
| + }
|
| + if( rc!=SQLITE_OK ){
|
| + fts5AsciiDelete((Fts5Tokenizer*)p);
|
| + p = 0;
|
| + }
|
| + }
|
| + }
|
| +
|
| + *ppOut = (Fts5Tokenizer*)p;
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +static void asciiFold(char *aOut, const char *aIn, int nByte){
|
| + int i;
|
| + for(i=0; i<nByte; i++){
|
| + char c = aIn[i];
|
| + if( c>='A' && c<='Z' ) c += 32;
|
| + aOut[i] = c;
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Tokenize some text using the ascii tokenizer.
|
| +*/
|
| +static int fts5AsciiTokenize(
|
| + Fts5Tokenizer *pTokenizer,
|
| + void *pCtx,
|
| + int flags,
|
| + const char *pText, int nText,
|
| + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
|
| +){
|
| + AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer;
|
| + int rc = SQLITE_OK;
|
| + int ie;
|
| + int is = 0;
|
| +
|
| + char aFold[64];
|
| + int nFold = sizeof(aFold);
|
| + char *pFold = aFold;
|
| + unsigned char *a = p->aTokenChar;
|
| +
|
| + while( is<nText && rc==SQLITE_OK ){
|
| + int nByte;
|
| +
|
| + /* Skip any leading divider characters. */
|
| + while( is<nText && ((pText[is]&0x80)==0 && a[(int)pText[is]]==0) ){
|
| + is++;
|
| + }
|
| + if( is==nText ) break;
|
| +
|
| + /* Count the token characters */
|
| + ie = is+1;
|
| + while( ie<nText && ((pText[ie]&0x80) || a[(int)pText[ie]] ) ){
|
| + ie++;
|
| + }
|
| +
|
| + /* Fold to lower case */
|
| + nByte = ie-is;
|
| + if( nByte>nFold ){
|
| + if( pFold!=aFold ) sqlite3_free(pFold);
|
| + pFold = sqlite3_malloc(nByte*2);
|
| + if( pFold==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + break;
|
| + }
|
| + nFold = nByte*2;
|
| + }
|
| + asciiFold(pFold, &pText[is], nByte);
|
| +
|
| + /* Invoke the token callback */
|
| + rc = xToken(pCtx, 0, pFold, nByte, is, ie);
|
| + is = ie+1;
|
| + }
|
| +
|
| + if( pFold!=aFold ) sqlite3_free(pFold);
|
| + if( rc==SQLITE_DONE ) rc = SQLITE_OK;
|
| + return rc;
|
| +}
|
| +
|
| +/**************************************************************************
|
| +** Start of unicode61 tokenizer implementation.
|
| +*/
|
| +
|
| +
|
| +/*
|
| +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied
|
| +** from the sqlite3 source file utf.c. If this file is compiled as part
|
| +** of the amalgamation, they are not required.
|
| +*/
|
| +#ifndef SQLITE_AMALGAMATION
|
| +
|
| +static const unsigned char sqlite3Utf8Trans1[] = {
|
| + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
| + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
|
| + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
|
| + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
|
| + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
| + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
|
| + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
| + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
|
| +};
|
| +
|
| +#define READ_UTF8(zIn, zTerm, c) \
|
| + c = *(zIn++); \
|
| + if( c>=0xc0 ){ \
|
| + c = sqlite3Utf8Trans1[c-0xc0]; \
|
| + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \
|
| + c = (c<<6) + (0x3f & *(zIn++)); \
|
| + } \
|
| + if( c<0x80 \
|
| + || (c&0xFFFFF800)==0xD800 \
|
| + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
|
| + }
|
| +
|
| +
|
| +#define WRITE_UTF8(zOut, c) { \
|
| + if( c<0x00080 ){ \
|
| + *zOut++ = (unsigned char)(c&0xFF); \
|
| + } \
|
| + else if( c<0x00800 ){ \
|
| + *zOut++ = 0xC0 + (unsigned char)((c>>6)&0x1F); \
|
| + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
|
| + } \
|
| + else if( c<0x10000 ){ \
|
| + *zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \
|
| + *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
|
| + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
|
| + }else{ \
|
| + *zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \
|
| + *zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \
|
| + *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
|
| + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
|
| + } \
|
| +}
|
| +
|
| +#endif /* ifndef SQLITE_AMALGAMATION */
|
| +
|
| +typedef struct Unicode61Tokenizer Unicode61Tokenizer;
|
| +struct Unicode61Tokenizer {
|
| + unsigned char aTokenChar[128]; /* ASCII range token characters */
|
| + char *aFold; /* Buffer to fold text into */
|
| + int nFold; /* Size of aFold[] in bytes */
|
| + int bRemoveDiacritic; /* True if remove_diacritics=1 is set */
|
| + int nException;
|
| + int *aiException;
|
| +};
|
| +
|
| +static int fts5UnicodeAddExceptions(
|
| + Unicode61Tokenizer *p, /* Tokenizer object */
|
| + const char *z, /* Characters to treat as exceptions */
|
| + int bTokenChars /* 1 for 'tokenchars', 0 for 'separators' */
|
| +){
|
| + int rc = SQLITE_OK;
|
| + int n = (int)strlen(z);
|
| + int *aNew;
|
| +
|
| + if( n>0 ){
|
| + aNew = (int*)sqlite3_realloc(p->aiException, (n+p->nException)*sizeof(int));
|
| + if( aNew ){
|
| + int nNew = p->nException;
|
| + const unsigned char *zCsr = (const unsigned char*)z;
|
| + const unsigned char *zTerm = (const unsigned char*)&z[n];
|
| + while( zCsr<zTerm ){
|
| + int iCode;
|
| + int bToken;
|
| + READ_UTF8(zCsr, zTerm, iCode);
|
| + if( iCode<128 ){
|
| + p->aTokenChar[iCode] = (unsigned char)bTokenChars;
|
| + }else{
|
| + bToken = sqlite3Fts5UnicodeIsalnum(iCode);
|
| + assert( (bToken==0 || bToken==1) );
|
| + assert( (bTokenChars==0 || bTokenChars==1) );
|
| + if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
|
| + int i;
|
| + for(i=0; i<nNew; i++){
|
| + if( aNew[i]>iCode ) break;
|
| + }
|
| + memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int));
|
| + aNew[i] = iCode;
|
| + nNew++;
|
| + }
|
| + }
|
| + }
|
| + p->aiException = aNew;
|
| + p->nException = nNew;
|
| + }else{
|
| + rc = SQLITE_NOMEM;
|
| + }
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Return true if the p->aiException[] array contains the value iCode.
|
| +*/
|
| +static int fts5UnicodeIsException(Unicode61Tokenizer *p, int iCode){
|
| + if( p->nException>0 ){
|
| + int *a = p->aiException;
|
| + int iLo = 0;
|
| + int iHi = p->nException-1;
|
| +
|
| + while( iHi>=iLo ){
|
| + int iTest = (iHi + iLo) / 2;
|
| + if( iCode==a[iTest] ){
|
| + return 1;
|
| + }else if( iCode>a[iTest] ){
|
| + iLo = iTest+1;
|
| + }else{
|
| + iHi = iTest-1;
|
| + }
|
| + }
|
| + }
|
| +
|
| + return 0;
|
| +}
|
| +
|
| +/*
|
| +** Delete a "unicode61" tokenizer.
|
| +*/
|
| +static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
|
| + if( pTok ){
|
| + Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok;
|
| + sqlite3_free(p->aiException);
|
| + sqlite3_free(p->aFold);
|
| + sqlite3_free(p);
|
| + }
|
| + return;
|
| +}
|
| +
|
| +/*
|
| +** Create a "unicode61" tokenizer.
|
| +*/
|
| +static int fts5UnicodeCreate(
|
| + void *pCtx,
|
| + const char **azArg, int nArg,
|
| + Fts5Tokenizer **ppOut
|
| +){
|
| + int rc = SQLITE_OK; /* Return code */
|
| + Unicode61Tokenizer *p = 0; /* New tokenizer object */
|
| +
|
| + if( nArg%2 ){
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
|
| + if( p ){
|
| + int i;
|
| + memset(p, 0, sizeof(Unicode61Tokenizer));
|
| + memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
|
| + p->bRemoveDiacritic = 1;
|
| + p->nFold = 64;
|
| + p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
|
| + if( p->aFold==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }
|
| + for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
|
| + const char *zArg = azArg[i+1];
|
| + if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
|
| + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
|
| + rc = SQLITE_ERROR;
|
| + }
|
| + p->bRemoveDiacritic = (zArg[0]=='1');
|
| + }else
|
| + if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
|
| + rc = fts5UnicodeAddExceptions(p, zArg, 1);
|
| + }else
|
| + if( 0==sqlite3_stricmp(azArg[i], "separators") ){
|
| + rc = fts5UnicodeAddExceptions(p, zArg, 0);
|
| + }else{
|
| + rc = SQLITE_ERROR;
|
| + }
|
| + }
|
| + }else{
|
| + rc = SQLITE_NOMEM;
|
| + }
|
| + if( rc!=SQLITE_OK ){
|
| + fts5UnicodeDelete((Fts5Tokenizer*)p);
|
| + p = 0;
|
| + }
|
| + *ppOut = (Fts5Tokenizer*)p;
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** Return true if, for the purposes of tokenizing with the tokenizer
|
| +** passed as the first argument, codepoint iCode is considered a token
|
| +** character (not a separator).
|
| +*/
|
| +static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
|
| + assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
|
| + return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode);
|
| +}
|
| +
|
| +static int fts5UnicodeTokenize(
|
| + Fts5Tokenizer *pTokenizer,
|
| + void *pCtx,
|
| + int flags,
|
| + const char *pText, int nText,
|
| + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
|
| +){
|
| + Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer;
|
| + int rc = SQLITE_OK;
|
| + unsigned char *a = p->aTokenChar;
|
| +
|
| + unsigned char *zTerm = (unsigned char*)&pText[nText];
|
| + unsigned char *zCsr = (unsigned char *)pText;
|
| +
|
| + /* Output buffer */
|
| + char *aFold = p->aFold;
|
| + int nFold = p->nFold;
|
| + const char *pEnd = &aFold[nFold-6];
|
| +
|
| + /* Each iteration of this loop gobbles up a contiguous run of separators,
|
| + ** then the next token. */
|
| + while( rc==SQLITE_OK ){
|
| + int iCode; /* non-ASCII codepoint read from input */
|
| + char *zOut = aFold;
|
| + int is;
|
| + int ie;
|
| +
|
| + /* Skip any separator characters. */
|
| + while( 1 ){
|
| + if( zCsr>=zTerm ) goto tokenize_done;
|
| + if( *zCsr & 0x80 ) {
|
| + /* A character outside of the ascii range. Skip past it if it is
|
| + ** a separator character. Or break out of the loop if it is not. */
|
| + is = zCsr - (unsigned char*)pText;
|
| + READ_UTF8(zCsr, zTerm, iCode);
|
| + if( fts5UnicodeIsAlnum(p, iCode) ){
|
| + goto non_ascii_tokenchar;
|
| + }
|
| + }else{
|
| + if( a[*zCsr] ){
|
| + is = zCsr - (unsigned char*)pText;
|
| + goto ascii_tokenchar;
|
| + }
|
| + zCsr++;
|
| + }
|
| + }
|
| +
|
| + /* Run through the tokenchars. Fold them into the output buffer along
|
| + ** the way. */
|
| + while( zCsr<zTerm ){
|
| +
|
| + /* Grow the output buffer so that there is sufficient space to fit the
|
| + ** largest possible utf-8 character. */
|
| + if( zOut>pEnd ){
|
| + aFold = sqlite3_malloc(nFold*2);
|
| + if( aFold==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + goto tokenize_done;
|
| + }
|
| + zOut = &aFold[zOut - p->aFold];
|
| + memcpy(aFold, p->aFold, nFold);
|
| + sqlite3_free(p->aFold);
|
| + p->aFold = aFold;
|
| + p->nFold = nFold = nFold*2;
|
| + pEnd = &aFold[nFold-6];
|
| + }
|
| +
|
| + if( *zCsr & 0x80 ){
|
| + /* An non-ascii-range character. Fold it into the output buffer if
|
| + ** it is a token character, or break out of the loop if it is not. */
|
| + READ_UTF8(zCsr, zTerm, iCode);
|
| + if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
|
| + non_ascii_tokenchar:
|
| + iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
|
| + if( iCode ) WRITE_UTF8(zOut, iCode);
|
| + }else{
|
| + break;
|
| + }
|
| + }else if( a[*zCsr]==0 ){
|
| + /* An ascii-range separator character. End of token. */
|
| + break;
|
| + }else{
|
| + ascii_tokenchar:
|
| + if( *zCsr>='A' && *zCsr<='Z' ){
|
| + *zOut++ = *zCsr + 32;
|
| + }else{
|
| + *zOut++ = *zCsr;
|
| + }
|
| + zCsr++;
|
| + }
|
| + ie = zCsr - (unsigned char*)pText;
|
| + }
|
| +
|
| + /* Invoke the token callback */
|
| + rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie);
|
| + }
|
| +
|
| + tokenize_done:
|
| + if( rc==SQLITE_DONE ) rc = SQLITE_OK;
|
| + return rc;
|
| +}
|
| +
|
| +/**************************************************************************
|
| +** Start of porter stemmer implementation.
|
| +*/
|
| +
|
| +/* Any tokens larger than this (in bytes) are passed through without
|
| +** stemming. */
|
| +#define FTS5_PORTER_MAX_TOKEN 64
|
| +
|
| +typedef struct PorterTokenizer PorterTokenizer;
|
| +struct PorterTokenizer {
|
| + fts5_tokenizer tokenizer; /* Parent tokenizer module */
|
| + Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */
|
| + char aBuf[FTS5_PORTER_MAX_TOKEN + 64];
|
| +};
|
| +
|
| +/*
|
| +** Delete a "porter" tokenizer.
|
| +*/
|
| +static void fts5PorterDelete(Fts5Tokenizer *pTok){
|
| + if( pTok ){
|
| + PorterTokenizer *p = (PorterTokenizer*)pTok;
|
| + if( p->pTokenizer ){
|
| + p->tokenizer.xDelete(p->pTokenizer);
|
| + }
|
| + sqlite3_free(p);
|
| + }
|
| +}
|
| +
|
| +/*
|
| +** Create a "porter" tokenizer.
|
| +*/
|
| +static int fts5PorterCreate(
|
| + void *pCtx,
|
| + const char **azArg, int nArg,
|
| + Fts5Tokenizer **ppOut
|
| +){
|
| + fts5_api *pApi = (fts5_api*)pCtx;
|
| + int rc = SQLITE_OK;
|
| + PorterTokenizer *pRet;
|
| + void *pUserdata = 0;
|
| + const char *zBase = "unicode61";
|
| +
|
| + if( nArg>0 ){
|
| + zBase = azArg[0];
|
| + }
|
| +
|
| + pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
|
| + if( pRet ){
|
| + memset(pRet, 0, sizeof(PorterTokenizer));
|
| + rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
|
| + }else{
|
| + rc = SQLITE_NOMEM;
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + int nArg2 = (nArg>0 ? nArg-1 : 0);
|
| + const char **azArg2 = (nArg2 ? &azArg[1] : 0);
|
| + rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer);
|
| + }
|
| +
|
| + if( rc!=SQLITE_OK ){
|
| + fts5PorterDelete((Fts5Tokenizer*)pRet);
|
| + pRet = 0;
|
| + }
|
| + *ppOut = (Fts5Tokenizer*)pRet;
|
| + return rc;
|
| +}
|
| +
|
| +typedef struct PorterContext PorterContext;
|
| +struct PorterContext {
|
| + void *pCtx;
|
| + int (*xToken)(void*, int, const char*, int, int, int);
|
| + char *aBuf;
|
| +};
|
| +
|
| +typedef struct PorterRule PorterRule;
|
| +struct PorterRule {
|
| + const char *zSuffix;
|
| + int nSuffix;
|
| + int (*xCond)(char *zStem, int nStem);
|
| + const char *zOutput;
|
| + int nOutput;
|
| +};
|
| +
|
| +#if 0
|
| +static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){
|
| + int ret = -1;
|
| + int nBuf = *pnBuf;
|
| + PorterRule *p;
|
| +
|
| + for(p=aRule; p->zSuffix; p++){
|
| + assert( strlen(p->zSuffix)==p->nSuffix );
|
| + assert( strlen(p->zOutput)==p->nOutput );
|
| + if( nBuf<p->nSuffix ) continue;
|
| + if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break;
|
| + }
|
| +
|
| + if( p->zSuffix ){
|
| + int nStem = nBuf - p->nSuffix;
|
| + if( p->xCond==0 || p->xCond(aBuf, nStem) ){
|
| + memcpy(&aBuf[nStem], p->zOutput, p->nOutput);
|
| + *pnBuf = nStem + p->nOutput;
|
| + ret = p - aRule;
|
| + }
|
| + }
|
| +
|
| + return ret;
|
| +}
|
| +#endif
|
| +
|
| +static int fts5PorterIsVowel(char c, int bYIsVowel){
|
| + return (
|
| + c=='a' || c=='e' || c=='i' || c=='o' || c=='u' || (bYIsVowel && c=='y')
|
| + );
|
| +}
|
| +
|
| +static int fts5PorterGobbleVC(char *zStem, int nStem, int bPrevCons){
|
| + int i;
|
| + int bCons = bPrevCons;
|
| +
|
| + /* Scan for a vowel */
|
| + for(i=0; i<nStem; i++){
|
| + if( 0==(bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) break;
|
| + }
|
| +
|
| + /* Scan for a consonent */
|
| + for(i++; i<nStem; i++){
|
| + if( (bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) return i+1;
|
| + }
|
| + return 0;
|
| +}
|
| +
|
| +/* porter rule condition: (m > 0) */
|
| +static int fts5Porter_MGt0(char *zStem, int nStem){
|
| + return !!fts5PorterGobbleVC(zStem, nStem, 0);
|
| +}
|
| +
|
| +/* porter rule condition: (m > 1) */
|
| +static int fts5Porter_MGt1(char *zStem, int nStem){
|
| + int n;
|
| + n = fts5PorterGobbleVC(zStem, nStem, 0);
|
| + if( n && fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
|
| + return 1;
|
| + }
|
| + return 0;
|
| +}
|
| +
|
| +/* porter rule condition: (m = 1) */
|
| +static int fts5Porter_MEq1(char *zStem, int nStem){
|
| + int n;
|
| + n = fts5PorterGobbleVC(zStem, nStem, 0);
|
| + if( n && 0==fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
|
| + return 1;
|
| + }
|
| + return 0;
|
| +}
|
| +
|
| +/* porter rule condition: (*o) */
|
| +static int fts5Porter_Ostar(char *zStem, int nStem){
|
| + if( zStem[nStem-1]=='w' || zStem[nStem-1]=='x' || zStem[nStem-1]=='y' ){
|
| + return 0;
|
| + }else{
|
| + int i;
|
| + int mask = 0;
|
| + int bCons = 0;
|
| + for(i=0; i<nStem; i++){
|
| + bCons = !fts5PorterIsVowel(zStem[i], bCons);
|
| + assert( bCons==0 || bCons==1 );
|
| + mask = (mask << 1) + bCons;
|
| + }
|
| + return ((mask & 0x0007)==0x0005);
|
| + }
|
| +}
|
| +
|
| +/* porter rule condition: (m > 1 and (*S or *T)) */
|
| +static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){
|
| + assert( nStem>0 );
|
| + return (zStem[nStem-1]=='s' || zStem[nStem-1]=='t')
|
| + && fts5Porter_MGt1(zStem, nStem);
|
| +}
|
| +
|
| +/* porter rule condition: (*v*) */
|
| +static int fts5Porter_Vowel(char *zStem, int nStem){
|
| + int i;
|
| + for(i=0; i<nStem; i++){
|
| + if( fts5PorterIsVowel(zStem[i], i>0) ){
|
| + return 1;
|
| + }
|
| + }
|
| + return 0;
|
| +}
|
| +
|
| +
|
| +/**************************************************************************
|
| +***************************************************************************
|
| +** GENERATED CODE STARTS HERE (mkportersteps.tcl)
|
| +*/
|
| +
|
| +static int fts5PorterStep4(char *aBuf, int *pnBuf){
|
| + int ret = 0;
|
| + int nBuf = *pnBuf;
|
| + switch( aBuf[nBuf-2] ){
|
| +
|
| + case 'a':
|
| + if( nBuf>2 && 0==memcmp("al", &aBuf[nBuf-2], 2) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-2) ){
|
| + *pnBuf = nBuf - 2;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'c':
|
| + if( nBuf>4 && 0==memcmp("ance", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-4) ){
|
| + *pnBuf = nBuf - 4;
|
| + }
|
| + }else if( nBuf>4 && 0==memcmp("ence", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-4) ){
|
| + *pnBuf = nBuf - 4;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'e':
|
| + if( nBuf>2 && 0==memcmp("er", &aBuf[nBuf-2], 2) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-2) ){
|
| + *pnBuf = nBuf - 2;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'i':
|
| + if( nBuf>2 && 0==memcmp("ic", &aBuf[nBuf-2], 2) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-2) ){
|
| + *pnBuf = nBuf - 2;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'l':
|
| + if( nBuf>4 && 0==memcmp("able", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-4) ){
|
| + *pnBuf = nBuf - 4;
|
| + }
|
| + }else if( nBuf>4 && 0==memcmp("ible", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-4) ){
|
| + *pnBuf = nBuf - 4;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'n':
|
| + if( nBuf>3 && 0==memcmp("ant", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-3) ){
|
| + *pnBuf = nBuf - 3;
|
| + }
|
| + }else if( nBuf>5 && 0==memcmp("ement", &aBuf[nBuf-5], 5) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-5) ){
|
| + *pnBuf = nBuf - 5;
|
| + }
|
| + }else if( nBuf>4 && 0==memcmp("ment", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-4) ){
|
| + *pnBuf = nBuf - 4;
|
| + }
|
| + }else if( nBuf>3 && 0==memcmp("ent", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-3) ){
|
| + *pnBuf = nBuf - 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'o':
|
| + if( nBuf>3 && 0==memcmp("ion", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt1_and_S_or_T(aBuf, nBuf-3) ){
|
| + *pnBuf = nBuf - 3;
|
| + }
|
| + }else if( nBuf>2 && 0==memcmp("ou", &aBuf[nBuf-2], 2) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-2) ){
|
| + *pnBuf = nBuf - 2;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 's':
|
| + if( nBuf>3 && 0==memcmp("ism", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-3) ){
|
| + *pnBuf = nBuf - 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 't':
|
| + if( nBuf>3 && 0==memcmp("ate", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-3) ){
|
| + *pnBuf = nBuf - 3;
|
| + }
|
| + }else if( nBuf>3 && 0==memcmp("iti", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-3) ){
|
| + *pnBuf = nBuf - 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'u':
|
| + if( nBuf>3 && 0==memcmp("ous", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-3) ){
|
| + *pnBuf = nBuf - 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'v':
|
| + if( nBuf>3 && 0==memcmp("ive", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-3) ){
|
| + *pnBuf = nBuf - 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'z':
|
| + if( nBuf>3 && 0==memcmp("ize", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-3) ){
|
| + *pnBuf = nBuf - 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + }
|
| + return ret;
|
| +}
|
| +
|
| +
|
| +static int fts5PorterStep1B2(char *aBuf, int *pnBuf){
|
| + int ret = 0;
|
| + int nBuf = *pnBuf;
|
| + switch( aBuf[nBuf-2] ){
|
| +
|
| + case 'a':
|
| + if( nBuf>2 && 0==memcmp("at", &aBuf[nBuf-2], 2) ){
|
| + memcpy(&aBuf[nBuf-2], "ate", 3);
|
| + *pnBuf = nBuf - 2 + 3;
|
| + ret = 1;
|
| + }
|
| + break;
|
| +
|
| + case 'b':
|
| + if( nBuf>2 && 0==memcmp("bl", &aBuf[nBuf-2], 2) ){
|
| + memcpy(&aBuf[nBuf-2], "ble", 3);
|
| + *pnBuf = nBuf - 2 + 3;
|
| + ret = 1;
|
| + }
|
| + break;
|
| +
|
| + case 'i':
|
| + if( nBuf>2 && 0==memcmp("iz", &aBuf[nBuf-2], 2) ){
|
| + memcpy(&aBuf[nBuf-2], "ize", 3);
|
| + *pnBuf = nBuf - 2 + 3;
|
| + ret = 1;
|
| + }
|
| + break;
|
| +
|
| + }
|
| + return ret;
|
| +}
|
| +
|
| +
|
| +static int fts5PorterStep2(char *aBuf, int *pnBuf){
|
| + int ret = 0;
|
| + int nBuf = *pnBuf;
|
| + switch( aBuf[nBuf-2] ){
|
| +
|
| + case 'a':
|
| + if( nBuf>7 && 0==memcmp("ational", &aBuf[nBuf-7], 7) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-7) ){
|
| + memcpy(&aBuf[nBuf-7], "ate", 3);
|
| + *pnBuf = nBuf - 7 + 3;
|
| + }
|
| + }else if( nBuf>6 && 0==memcmp("tional", &aBuf[nBuf-6], 6) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-6) ){
|
| + memcpy(&aBuf[nBuf-6], "tion", 4);
|
| + *pnBuf = nBuf - 6 + 4;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'c':
|
| + if( nBuf>4 && 0==memcmp("enci", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-4) ){
|
| + memcpy(&aBuf[nBuf-4], "ence", 4);
|
| + *pnBuf = nBuf - 4 + 4;
|
| + }
|
| + }else if( nBuf>4 && 0==memcmp("anci", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-4) ){
|
| + memcpy(&aBuf[nBuf-4], "ance", 4);
|
| + *pnBuf = nBuf - 4 + 4;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'e':
|
| + if( nBuf>4 && 0==memcmp("izer", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-4) ){
|
| + memcpy(&aBuf[nBuf-4], "ize", 3);
|
| + *pnBuf = nBuf - 4 + 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'g':
|
| + if( nBuf>4 && 0==memcmp("logi", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-4) ){
|
| + memcpy(&aBuf[nBuf-4], "log", 3);
|
| + *pnBuf = nBuf - 4 + 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'l':
|
| + if( nBuf>3 && 0==memcmp("bli", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-3) ){
|
| + memcpy(&aBuf[nBuf-3], "ble", 3);
|
| + *pnBuf = nBuf - 3 + 3;
|
| + }
|
| + }else if( nBuf>4 && 0==memcmp("alli", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-4) ){
|
| + memcpy(&aBuf[nBuf-4], "al", 2);
|
| + *pnBuf = nBuf - 4 + 2;
|
| + }
|
| + }else if( nBuf>5 && 0==memcmp("entli", &aBuf[nBuf-5], 5) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-5) ){
|
| + memcpy(&aBuf[nBuf-5], "ent", 3);
|
| + *pnBuf = nBuf - 5 + 3;
|
| + }
|
| + }else if( nBuf>3 && 0==memcmp("eli", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-3) ){
|
| + memcpy(&aBuf[nBuf-3], "e", 1);
|
| + *pnBuf = nBuf - 3 + 1;
|
| + }
|
| + }else if( nBuf>5 && 0==memcmp("ousli", &aBuf[nBuf-5], 5) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-5) ){
|
| + memcpy(&aBuf[nBuf-5], "ous", 3);
|
| + *pnBuf = nBuf - 5 + 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'o':
|
| + if( nBuf>7 && 0==memcmp("ization", &aBuf[nBuf-7], 7) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-7) ){
|
| + memcpy(&aBuf[nBuf-7], "ize", 3);
|
| + *pnBuf = nBuf - 7 + 3;
|
| + }
|
| + }else if( nBuf>5 && 0==memcmp("ation", &aBuf[nBuf-5], 5) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-5) ){
|
| + memcpy(&aBuf[nBuf-5], "ate", 3);
|
| + *pnBuf = nBuf - 5 + 3;
|
| + }
|
| + }else if( nBuf>4 && 0==memcmp("ator", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-4) ){
|
| + memcpy(&aBuf[nBuf-4], "ate", 3);
|
| + *pnBuf = nBuf - 4 + 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 's':
|
| + if( nBuf>5 && 0==memcmp("alism", &aBuf[nBuf-5], 5) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-5) ){
|
| + memcpy(&aBuf[nBuf-5], "al", 2);
|
| + *pnBuf = nBuf - 5 + 2;
|
| + }
|
| + }else if( nBuf>7 && 0==memcmp("iveness", &aBuf[nBuf-7], 7) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-7) ){
|
| + memcpy(&aBuf[nBuf-7], "ive", 3);
|
| + *pnBuf = nBuf - 7 + 3;
|
| + }
|
| + }else if( nBuf>7 && 0==memcmp("fulness", &aBuf[nBuf-7], 7) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-7) ){
|
| + memcpy(&aBuf[nBuf-7], "ful", 3);
|
| + *pnBuf = nBuf - 7 + 3;
|
| + }
|
| + }else if( nBuf>7 && 0==memcmp("ousness", &aBuf[nBuf-7], 7) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-7) ){
|
| + memcpy(&aBuf[nBuf-7], "ous", 3);
|
| + *pnBuf = nBuf - 7 + 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 't':
|
| + if( nBuf>5 && 0==memcmp("aliti", &aBuf[nBuf-5], 5) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-5) ){
|
| + memcpy(&aBuf[nBuf-5], "al", 2);
|
| + *pnBuf = nBuf - 5 + 2;
|
| + }
|
| + }else if( nBuf>5 && 0==memcmp("iviti", &aBuf[nBuf-5], 5) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-5) ){
|
| + memcpy(&aBuf[nBuf-5], "ive", 3);
|
| + *pnBuf = nBuf - 5 + 3;
|
| + }
|
| + }else if( nBuf>6 && 0==memcmp("biliti", &aBuf[nBuf-6], 6) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-6) ){
|
| + memcpy(&aBuf[nBuf-6], "ble", 3);
|
| + *pnBuf = nBuf - 6 + 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + }
|
| + return ret;
|
| +}
|
| +
|
| +
|
| +static int fts5PorterStep3(char *aBuf, int *pnBuf){
|
| + int ret = 0;
|
| + int nBuf = *pnBuf;
|
| + switch( aBuf[nBuf-2] ){
|
| +
|
| + case 'a':
|
| + if( nBuf>4 && 0==memcmp("ical", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-4) ){
|
| + memcpy(&aBuf[nBuf-4], "ic", 2);
|
| + *pnBuf = nBuf - 4 + 2;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 's':
|
| + if( nBuf>4 && 0==memcmp("ness", &aBuf[nBuf-4], 4) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-4) ){
|
| + *pnBuf = nBuf - 4;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 't':
|
| + if( nBuf>5 && 0==memcmp("icate", &aBuf[nBuf-5], 5) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-5) ){
|
| + memcpy(&aBuf[nBuf-5], "ic", 2);
|
| + *pnBuf = nBuf - 5 + 2;
|
| + }
|
| + }else if( nBuf>5 && 0==memcmp("iciti", &aBuf[nBuf-5], 5) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-5) ){
|
| + memcpy(&aBuf[nBuf-5], "ic", 2);
|
| + *pnBuf = nBuf - 5 + 2;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'u':
|
| + if( nBuf>3 && 0==memcmp("ful", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-3) ){
|
| + *pnBuf = nBuf - 3;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'v':
|
| + if( nBuf>5 && 0==memcmp("ative", &aBuf[nBuf-5], 5) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-5) ){
|
| + *pnBuf = nBuf - 5;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'z':
|
| + if( nBuf>5 && 0==memcmp("alize", &aBuf[nBuf-5], 5) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-5) ){
|
| + memcpy(&aBuf[nBuf-5], "al", 2);
|
| + *pnBuf = nBuf - 5 + 2;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + }
|
| + return ret;
|
| +}
|
| +
|
| +
|
| +static int fts5PorterStep1B(char *aBuf, int *pnBuf){
|
| + int ret = 0;
|
| + int nBuf = *pnBuf;
|
| + switch( aBuf[nBuf-2] ){
|
| +
|
| + case 'e':
|
| + if( nBuf>3 && 0==memcmp("eed", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_MGt0(aBuf, nBuf-3) ){
|
| + memcpy(&aBuf[nBuf-3], "ee", 2);
|
| + *pnBuf = nBuf - 3 + 2;
|
| + }
|
| + }else if( nBuf>2 && 0==memcmp("ed", &aBuf[nBuf-2], 2) ){
|
| + if( fts5Porter_Vowel(aBuf, nBuf-2) ){
|
| + *pnBuf = nBuf - 2;
|
| + ret = 1;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + case 'n':
|
| + if( nBuf>3 && 0==memcmp("ing", &aBuf[nBuf-3], 3) ){
|
| + if( fts5Porter_Vowel(aBuf, nBuf-3) ){
|
| + *pnBuf = nBuf - 3;
|
| + ret = 1;
|
| + }
|
| + }
|
| + break;
|
| +
|
| + }
|
| + return ret;
|
| +}
|
| +
|
| +/*
|
| +** GENERATED CODE ENDS HERE (mkportersteps.tcl)
|
| +***************************************************************************
|
| +**************************************************************************/
|
| +
|
| +static void fts5PorterStep1A(char *aBuf, int *pnBuf){
|
| + int nBuf = *pnBuf;
|
| + if( aBuf[nBuf-1]=='s' ){
|
| + if( aBuf[nBuf-2]=='e' ){
|
| + if( (nBuf>4 && aBuf[nBuf-4]=='s' && aBuf[nBuf-3]=='s')
|
| + || (nBuf>3 && aBuf[nBuf-3]=='i' )
|
| + ){
|
| + *pnBuf = nBuf-2;
|
| + }else{
|
| + *pnBuf = nBuf-1;
|
| + }
|
| + }
|
| + else if( aBuf[nBuf-2]!='s' ){
|
| + *pnBuf = nBuf-1;
|
| + }
|
| + }
|
| +}
|
| +
|
| +static int fts5PorterCb(
|
| + void *pCtx,
|
| + int tflags,
|
| + const char *pToken,
|
| + int nToken,
|
| + int iStart,
|
| + int iEnd
|
| +){
|
| + PorterContext *p = (PorterContext*)pCtx;
|
| +
|
| + char *aBuf;
|
| + int nBuf;
|
| +
|
| + if( nToken>FTS5_PORTER_MAX_TOKEN || nToken<3 ) goto pass_through;
|
| + aBuf = p->aBuf;
|
| + nBuf = nToken;
|
| + memcpy(aBuf, pToken, nBuf);
|
| +
|
| + /* Step 1. */
|
| + fts5PorterStep1A(aBuf, &nBuf);
|
| + if( fts5PorterStep1B(aBuf, &nBuf) ){
|
| + if( fts5PorterStep1B2(aBuf, &nBuf)==0 ){
|
| + char c = aBuf[nBuf-1];
|
| + if( fts5PorterIsVowel(c, 0)==0
|
| + && c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2]
|
| + ){
|
| + nBuf--;
|
| + }else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){
|
| + aBuf[nBuf++] = 'e';
|
| + }
|
| + }
|
| + }
|
| +
|
| + /* Step 1C. */
|
| + if( aBuf[nBuf-1]=='y' && fts5Porter_Vowel(aBuf, nBuf-1) ){
|
| + aBuf[nBuf-1] = 'i';
|
| + }
|
| +
|
| + /* Steps 2 through 4. */
|
| + fts5PorterStep2(aBuf, &nBuf);
|
| + fts5PorterStep3(aBuf, &nBuf);
|
| + fts5PorterStep4(aBuf, &nBuf);
|
| +
|
| + /* Step 5a. */
|
| + assert( nBuf>0 );
|
| + if( aBuf[nBuf-1]=='e' ){
|
| + if( fts5Porter_MGt1(aBuf, nBuf-1)
|
| + || (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1))
|
| + ){
|
| + nBuf--;
|
| + }
|
| + }
|
| +
|
| + /* Step 5b. */
|
| + if( nBuf>1 && aBuf[nBuf-1]=='l'
|
| + && aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1)
|
| + ){
|
| + nBuf--;
|
| + }
|
| +
|
| + return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd);
|
| +
|
| + pass_through:
|
| + return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
|
| +}
|
| +
|
| +/*
|
| +** Tokenize using the porter tokenizer.
|
| +*/
|
| +static int fts5PorterTokenize(
|
| + Fts5Tokenizer *pTokenizer,
|
| + void *pCtx,
|
| + int flags,
|
| + const char *pText, int nText,
|
| + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
|
| +){
|
| + PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
|
| + PorterContext sCtx;
|
| + sCtx.xToken = xToken;
|
| + sCtx.pCtx = pCtx;
|
| + sCtx.aBuf = p->aBuf;
|
| + return p->tokenizer.xTokenize(
|
| + p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb
|
| + );
|
| +}
|
| +
|
| +/*
|
| +** Register all built-in tokenizers with FTS5.
|
| +*/
|
| +static int sqlite3Fts5TokenizerInit(fts5_api *pApi){
|
| + struct BuiltinTokenizer {
|
| + const char *zName;
|
| + fts5_tokenizer x;
|
| + } aBuiltin[] = {
|
| + { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
|
| + { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
|
| + { "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
|
| + };
|
| +
|
| + int rc = SQLITE_OK; /* Return code */
|
| + int i; /* To iterate through builtin functions */
|
| +
|
| + for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){
|
| + rc = pApi->xCreateTokenizer(pApi,
|
| + aBuiltin[i].zName,
|
| + (void*)pApi,
|
| + &aBuiltin[i].x,
|
| + 0
|
| + );
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +
|
| +/*
|
| +** 2012 May 25
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +*/
|
| +
|
| +/*
|
| +** DO NOT EDIT THIS MACHINE GENERATED FILE.
|
| +*/
|
| +
|
| +
|
| +/* #include <assert.h> */
|
| +
|
| +/*
|
| +** Return true if the argument corresponds to a unicode codepoint
|
| +** classified as either a letter or a number. Otherwise false.
|
| +**
|
| +** The results are undefined if the value passed to this function
|
| +** is less than zero.
|
| +*/
|
| +static int sqlite3Fts5UnicodeIsalnum(int c){
|
| + /* Each unsigned integer in the following array corresponds to a contiguous
|
| + ** range of unicode codepoints that are not either letters or numbers (i.e.
|
| + ** codepoints for which this function should return 0).
|
| + **
|
| + ** The most significant 22 bits in each 32-bit value contain the first
|
| + ** codepoint in the range. The least significant 10 bits are used to store
|
| + ** the size of the range (always at least 1). In other words, the value
|
| + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint
|
| + ** C. It is not possible to represent a range larger than 1023 codepoints
|
| + ** using this format.
|
| + */
|
| + static const unsigned int aEntry[] = {
|
| + 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
|
| + 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
|
| + 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
|
| + 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
|
| + 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
|
| + 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
|
| + 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
|
| + 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401,
|
| + 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804,
|
| + 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403,
|
| + 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812,
|
| + 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001,
|
| + 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802,
|
| + 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805,
|
| + 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401,
|
| + 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03,
|
| + 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807,
|
| + 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001,
|
| + 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01,
|
| + 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804,
|
| + 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001,
|
| + 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802,
|
| + 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01,
|
| + 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06,
|
| + 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007,
|
| + 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006,
|
| + 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417,
|
| + 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14,
|
| + 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07,
|
| + 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01,
|
| + 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001,
|
| + 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802,
|
| + 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F,
|
| + 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002,
|
| + 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802,
|
| + 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006,
|
| + 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D,
|
| + 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802,
|
| + 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027,
|
| + 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403,
|
| + 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805,
|
| + 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04,
|
| + 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401,
|
| + 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005,
|
| + 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B,
|
| + 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A,
|
| + 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001,
|
| + 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59,
|
| + 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807,
|
| + 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01,
|
| + 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E,
|
| + 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100,
|
| + 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10,
|
| + 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402,
|
| + 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804,
|
| + 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012,
|
| + 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004,
|
| + 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002,
|
| + 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
|
| + 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
|
| + 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
|
| + 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
|
| + 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
|
| + 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
|
| + 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
|
| + 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
|
| + 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
|
| + 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
|
| + 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
|
| + 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
|
| + 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
|
| + 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
|
| + 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
|
| + 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
|
| + 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
|
| + 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
|
| + 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
|
| + 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
|
| + 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
|
| + 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
|
| + 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
|
| + 0x380400F0,
|
| + };
|
| + static const unsigned int aAscii[4] = {
|
| + 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
|
| + };
|
| +
|
| + if( c<128 ){
|
| + return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
|
| + }else if( c<(1<<22) ){
|
| + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
|
| + int iRes = 0;
|
| + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
|
| + int iLo = 0;
|
| + while( iHi>=iLo ){
|
| + int iTest = (iHi + iLo) / 2;
|
| + if( key >= aEntry[iTest] ){
|
| + iRes = iTest;
|
| + iLo = iTest+1;
|
| + }else{
|
| + iHi = iTest-1;
|
| + }
|
| + }
|
| + assert( aEntry[0]<key );
|
| + assert( key>=aEntry[iRes] );
|
| + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
|
| + }
|
| + return 1;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** If the argument is a codepoint corresponding to a lowercase letter
|
| +** in the ASCII range with a diacritic added, return the codepoint
|
| +** of the ASCII letter only. For example, if passed 235 - "LATIN
|
| +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
|
| +** E"). The resuls of passing a codepoint that corresponds to an
|
| +** uppercase letter are undefined.
|
| +*/
|
| +static int fts5_remove_diacritic(int c){
|
| + unsigned short aDia[] = {
|
| + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995,
|
| + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286,
|
| + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732,
|
| + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336,
|
| + 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928,
|
| + 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234,
|
| + 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504,
|
| + 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529,
|
| + 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726,
|
| + 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122,
|
| + 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536,
|
| + 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730,
|
| + 62924, 63050, 63082, 63274, 63390,
|
| + };
|
| + char aChar[] = {
|
| + '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c',
|
| + 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r',
|
| + 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o',
|
| + 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r',
|
| + 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0',
|
| + '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h',
|
| + 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't',
|
| + 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a',
|
| + 'e', 'i', 'o', 'u', 'y',
|
| + };
|
| +
|
| + unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
|
| + int iRes = 0;
|
| + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
|
| + int iLo = 0;
|
| + while( iHi>=iLo ){
|
| + int iTest = (iHi + iLo) / 2;
|
| + if( key >= aDia[iTest] ){
|
| + iRes = iTest;
|
| + iLo = iTest+1;
|
| + }else{
|
| + iHi = iTest-1;
|
| + }
|
| + }
|
| + assert( key>=aDia[iRes] );
|
| + return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Return true if the argument interpreted as a unicode codepoint
|
| +** is a diacritical modifier character.
|
| +*/
|
| +static int sqlite3Fts5UnicodeIsdiacritic(int c){
|
| + unsigned int mask0 = 0x08029FDF;
|
| + unsigned int mask1 = 0x000361F8;
|
| + if( c<768 || c>817 ) return 0;
|
| + return (c < 768+32) ?
|
| + (mask0 & (1 << (c-768))) :
|
| + (mask1 & (1 << (c-768-32)));
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Interpret the argument as a unicode codepoint. If the codepoint
|
| +** is an upper case character that has a lower case equivalent,
|
| +** return the codepoint corresponding to the lower case version.
|
| +** Otherwise, return a copy of the argument.
|
| +**
|
| +** The results are undefined if the value passed to this function
|
| +** is less than zero.
|
| +*/
|
| +static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
|
| + /* Each entry in the following array defines a rule for folding a range
|
| + ** of codepoints to lower case. The rule applies to a range of nRange
|
| + ** codepoints starting at codepoint iCode.
|
| + **
|
| + ** If the least significant bit in flags is clear, then the rule applies
|
| + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
|
| + ** need to be folded). Or, if it is set, then the rule only applies to
|
| + ** every second codepoint in the range, starting with codepoint C.
|
| + **
|
| + ** The 7 most significant bits in flags are an index into the aiOff[]
|
| + ** array. If a specific codepoint C does require folding, then its lower
|
| + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF).
|
| + **
|
| + ** The contents of this array are generated by parsing the CaseFolding.txt
|
| + ** file distributed as part of the "Unicode Character Database". See
|
| + ** http://www.unicode.org for details.
|
| + */
|
| + static const struct TableEntry {
|
| + unsigned short iCode;
|
| + unsigned char flags;
|
| + unsigned char nRange;
|
| + } aEntry[] = {
|
| + {65, 14, 26}, {181, 64, 1}, {192, 14, 23},
|
| + {216, 14, 7}, {256, 1, 48}, {306, 1, 6},
|
| + {313, 1, 16}, {330, 1, 46}, {376, 116, 1},
|
| + {377, 1, 6}, {383, 104, 1}, {385, 50, 1},
|
| + {386, 1, 4}, {390, 44, 1}, {391, 0, 1},
|
| + {393, 42, 2}, {395, 0, 1}, {398, 32, 1},
|
| + {399, 38, 1}, {400, 40, 1}, {401, 0, 1},
|
| + {403, 42, 1}, {404, 46, 1}, {406, 52, 1},
|
| + {407, 48, 1}, {408, 0, 1}, {412, 52, 1},
|
| + {413, 54, 1}, {415, 56, 1}, {416, 1, 6},
|
| + {422, 60, 1}, {423, 0, 1}, {425, 60, 1},
|
| + {428, 0, 1}, {430, 60, 1}, {431, 0, 1},
|
| + {433, 58, 2}, {435, 1, 4}, {439, 62, 1},
|
| + {440, 0, 1}, {444, 0, 1}, {452, 2, 1},
|
| + {453, 0, 1}, {455, 2, 1}, {456, 0, 1},
|
| + {458, 2, 1}, {459, 1, 18}, {478, 1, 18},
|
| + {497, 2, 1}, {498, 1, 4}, {502, 122, 1},
|
| + {503, 134, 1}, {504, 1, 40}, {544, 110, 1},
|
| + {546, 1, 18}, {570, 70, 1}, {571, 0, 1},
|
| + {573, 108, 1}, {574, 68, 1}, {577, 0, 1},
|
| + {579, 106, 1}, {580, 28, 1}, {581, 30, 1},
|
| + {582, 1, 10}, {837, 36, 1}, {880, 1, 4},
|
| + {886, 0, 1}, {902, 18, 1}, {904, 16, 3},
|
| + {908, 26, 1}, {910, 24, 2}, {913, 14, 17},
|
| + {931, 14, 9}, {962, 0, 1}, {975, 4, 1},
|
| + {976, 140, 1}, {977, 142, 1}, {981, 146, 1},
|
| + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1},
|
| + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1},
|
| + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1},
|
| + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32},
|
| + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1},
|
| + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38},
|
| + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1},
|
| + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1},
|
| + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6},
|
| + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6},
|
| + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8},
|
| + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2},
|
| + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1},
|
| + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2},
|
| + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2},
|
| + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2},
|
| + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1},
|
| + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16},
|
| + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47},
|
| + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1},
|
| + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1},
|
| + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1},
|
| + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2},
|
| + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1},
|
| + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14},
|
| + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1},
|
| + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1},
|
| + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1},
|
| + {65313, 14, 26},
|
| + };
|
| + static const unsigned short aiOff[] = {
|
| + 1, 2, 8, 15, 16, 26, 28, 32,
|
| + 37, 38, 40, 48, 63, 64, 69, 71,
|
| + 79, 80, 116, 202, 203, 205, 206, 207,
|
| + 209, 210, 211, 213, 214, 217, 218, 219,
|
| + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721,
|
| + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274,
|
| + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406,
|
| + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462,
|
| + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511,
|
| + 65514, 65521, 65527, 65528, 65529,
|
| + };
|
| +
|
| + int ret = c;
|
| +
|
| + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
|
| +
|
| + if( c<128 ){
|
| + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
|
| + }else if( c<65536 ){
|
| + const struct TableEntry *p;
|
| + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
|
| + int iLo = 0;
|
| + int iRes = -1;
|
| +
|
| + assert( c>aEntry[0].iCode );
|
| + while( iHi>=iLo ){
|
| + int iTest = (iHi + iLo) / 2;
|
| + int cmp = (c - aEntry[iTest].iCode);
|
| + if( cmp>=0 ){
|
| + iRes = iTest;
|
| + iLo = iTest+1;
|
| + }else{
|
| + iHi = iTest-1;
|
| + }
|
| + }
|
| +
|
| + assert( iRes>=0 && c>=aEntry[iRes].iCode );
|
| + p = &aEntry[iRes];
|
| + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
|
| + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
|
| + assert( ret>0 );
|
| + }
|
| +
|
| + if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
|
| + }
|
| +
|
| + else if( c>=66560 && c<66600 ){
|
| + ret = c + 40;
|
| + }
|
| +
|
| + return ret;
|
| +}
|
| +
|
| +/*
|
| +** 2015 May 30
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +** Routines for varint serialization and deserialization.
|
| +*/
|
| +
|
| +
|
| +/* #include "fts5Int.h" */
|
| +
|
| +/*
|
| +** This is a copy of the sqlite3GetVarint32() routine from the SQLite core.
|
| +** Except, this version does handle the single byte case that the core
|
| +** version depends on being handled before its function is called.
|
| +*/
|
| +static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){
|
| + u32 a,b;
|
| +
|
| + /* The 1-byte case. Overwhelmingly the most common. */
|
| + a = *p;
|
| + /* a: p0 (unmasked) */
|
| + if (!(a&0x80))
|
| + {
|
| + /* Values between 0 and 127 */
|
| + *v = a;
|
| + return 1;
|
| + }
|
| +
|
| + /* The 2-byte case */
|
| + p++;
|
| + b = *p;
|
| + /* b: p1 (unmasked) */
|
| + if (!(b&0x80))
|
| + {
|
| + /* Values between 128 and 16383 */
|
| + a &= 0x7f;
|
| + a = a<<7;
|
| + *v = a | b;
|
| + return 2;
|
| + }
|
| +
|
| + /* The 3-byte case */
|
| + p++;
|
| + a = a<<14;
|
| + a |= *p;
|
| + /* a: p0<<14 | p2 (unmasked) */
|
| + if (!(a&0x80))
|
| + {
|
| + /* Values between 16384 and 2097151 */
|
| + a &= (0x7f<<14)|(0x7f);
|
| + b &= 0x7f;
|
| + b = b<<7;
|
| + *v = a | b;
|
| + return 3;
|
| + }
|
| +
|
| + /* A 32-bit varint is used to store size information in btrees.
|
| + ** Objects are rarely larger than 2MiB limit of a 3-byte varint.
|
| + ** A 3-byte varint is sufficient, for example, to record the size
|
| + ** of a 1048569-byte BLOB or string.
|
| + **
|
| + ** We only unroll the first 1-, 2-, and 3- byte cases. The very
|
| + ** rare larger cases can be handled by the slower 64-bit varint
|
| + ** routine.
|
| + */
|
| + {
|
| + u64 v64;
|
| + u8 n;
|
| + p -= 2;
|
| + n = sqlite3Fts5GetVarint(p, &v64);
|
| + *v = (u32)v64;
|
| + assert( n>3 && n<=9 );
|
| + return n;
|
| + }
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Bitmasks used by sqlite3GetVarint(). These precomputed constants
|
| +** are defined here rather than simply putting the constant expressions
|
| +** inline in order to work around bugs in the RVT compiler.
|
| +**
|
| +** SLOT_2_0 A mask for (0x7f<<14) | 0x7f
|
| +**
|
| +** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0
|
| +*/
|
| +#define SLOT_2_0 0x001fc07f
|
| +#define SLOT_4_2_0 0xf01fc07f
|
| +
|
| +/*
|
| +** Read a 64-bit variable-length integer from memory starting at p[0].
|
| +** Return the number of bytes read. The value is stored in *v.
|
| +*/
|
| +static u8 sqlite3Fts5GetVarint(const unsigned char *p, u64 *v){
|
| + u32 a,b,s;
|
| +
|
| + a = *p;
|
| + /* a: p0 (unmasked) */
|
| + if (!(a&0x80))
|
| + {
|
| + *v = a;
|
| + return 1;
|
| + }
|
| +
|
| + p++;
|
| + b = *p;
|
| + /* b: p1 (unmasked) */
|
| + if (!(b&0x80))
|
| + {
|
| + a &= 0x7f;
|
| + a = a<<7;
|
| + a |= b;
|
| + *v = a;
|
| + return 2;
|
| + }
|
| +
|
| + /* Verify that constants are precomputed correctly */
|
| + assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) );
|
| + assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) );
|
| +
|
| + p++;
|
| + a = a<<14;
|
| + a |= *p;
|
| + /* a: p0<<14 | p2 (unmasked) */
|
| + if (!(a&0x80))
|
| + {
|
| + a &= SLOT_2_0;
|
| + b &= 0x7f;
|
| + b = b<<7;
|
| + a |= b;
|
| + *v = a;
|
| + return 3;
|
| + }
|
| +
|
| + /* CSE1 from below */
|
| + a &= SLOT_2_0;
|
| + p++;
|
| + b = b<<14;
|
| + b |= *p;
|
| + /* b: p1<<14 | p3 (unmasked) */
|
| + if (!(b&0x80))
|
| + {
|
| + b &= SLOT_2_0;
|
| + /* moved CSE1 up */
|
| + /* a &= (0x7f<<14)|(0x7f); */
|
| + a = a<<7;
|
| + a |= b;
|
| + *v = a;
|
| + return 4;
|
| + }
|
| +
|
| + /* a: p0<<14 | p2 (masked) */
|
| + /* b: p1<<14 | p3 (unmasked) */
|
| + /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
|
| + /* moved CSE1 up */
|
| + /* a &= (0x7f<<14)|(0x7f); */
|
| + b &= SLOT_2_0;
|
| + s = a;
|
| + /* s: p0<<14 | p2 (masked) */
|
| +
|
| + p++;
|
| + a = a<<14;
|
| + a |= *p;
|
| + /* a: p0<<28 | p2<<14 | p4 (unmasked) */
|
| + if (!(a&0x80))
|
| + {
|
| + /* we can skip these cause they were (effectively) done above in calc'ing s */
|
| + /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
|
| + /* b &= (0x7f<<14)|(0x7f); */
|
| + b = b<<7;
|
| + a |= b;
|
| + s = s>>18;
|
| + *v = ((u64)s)<<32 | a;
|
| + return 5;
|
| + }
|
| +
|
| + /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
|
| + s = s<<7;
|
| + s |= b;
|
| + /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
|
| +
|
| + p++;
|
| + b = b<<14;
|
| + b |= *p;
|
| + /* b: p1<<28 | p3<<14 | p5 (unmasked) */
|
| + if (!(b&0x80))
|
| + {
|
| + /* we can skip this cause it was (effectively) done above in calc'ing s */
|
| + /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
|
| + a &= SLOT_2_0;
|
| + a = a<<7;
|
| + a |= b;
|
| + s = s>>18;
|
| + *v = ((u64)s)<<32 | a;
|
| + return 6;
|
| + }
|
| +
|
| + p++;
|
| + a = a<<14;
|
| + a |= *p;
|
| + /* a: p2<<28 | p4<<14 | p6 (unmasked) */
|
| + if (!(a&0x80))
|
| + {
|
| + a &= SLOT_4_2_0;
|
| + b &= SLOT_2_0;
|
| + b = b<<7;
|
| + a |= b;
|
| + s = s>>11;
|
| + *v = ((u64)s)<<32 | a;
|
| + return 7;
|
| + }
|
| +
|
| + /* CSE2 from below */
|
| + a &= SLOT_2_0;
|
| + p++;
|
| + b = b<<14;
|
| + b |= *p;
|
| + /* b: p3<<28 | p5<<14 | p7 (unmasked) */
|
| + if (!(b&0x80))
|
| + {
|
| + b &= SLOT_4_2_0;
|
| + /* moved CSE2 up */
|
| + /* a &= (0x7f<<14)|(0x7f); */
|
| + a = a<<7;
|
| + a |= b;
|
| + s = s>>4;
|
| + *v = ((u64)s)<<32 | a;
|
| + return 8;
|
| + }
|
| +
|
| + p++;
|
| + a = a<<15;
|
| + a |= *p;
|
| + /* a: p4<<29 | p6<<15 | p8 (unmasked) */
|
| +
|
| + /* moved CSE2 up */
|
| + /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */
|
| + b &= SLOT_2_0;
|
| + b = b<<8;
|
| + a |= b;
|
| +
|
| + s = s<<4;
|
| + b = p[-4];
|
| + b &= 0x7f;
|
| + b = b>>3;
|
| + s |= b;
|
| +
|
| + *v = ((u64)s)<<32 | a;
|
| +
|
| + return 9;
|
| +}
|
| +
|
| +/*
|
| +** The variable-length integer encoding is as follows:
|
| +**
|
| +** KEY:
|
| +** A = 0xxxxxxx 7 bits of data and one flag bit
|
| +** B = 1xxxxxxx 7 bits of data and one flag bit
|
| +** C = xxxxxxxx 8 bits of data
|
| +**
|
| +** 7 bits - A
|
| +** 14 bits - BA
|
| +** 21 bits - BBA
|
| +** 28 bits - BBBA
|
| +** 35 bits - BBBBA
|
| +** 42 bits - BBBBBA
|
| +** 49 bits - BBBBBBA
|
| +** 56 bits - BBBBBBBA
|
| +** 64 bits - BBBBBBBBC
|
| +*/
|
| +
|
| +#ifdef SQLITE_NOINLINE
|
| +# define FTS5_NOINLINE SQLITE_NOINLINE
|
| +#else
|
| +# define FTS5_NOINLINE
|
| +#endif
|
| +
|
| +/*
|
| +** Write a 64-bit variable-length integer to memory starting at p[0].
|
| +** The length of data write will be between 1 and 9 bytes. The number
|
| +** of bytes written is returned.
|
| +**
|
| +** A variable-length integer consists of the lower 7 bits of each byte
|
| +** for all bytes that have the 8th bit set and one byte with the 8th
|
| +** bit clear. Except, if we get to the 9th byte, it stores the full
|
| +** 8 bits and is the last byte.
|
| +*/
|
| +static int FTS5_NOINLINE fts5PutVarint64(unsigned char *p, u64 v){
|
| + int i, j, n;
|
| + u8 buf[10];
|
| + if( v & (((u64)0xff000000)<<32) ){
|
| + p[8] = (u8)v;
|
| + v >>= 8;
|
| + for(i=7; i>=0; i--){
|
| + p[i] = (u8)((v & 0x7f) | 0x80);
|
| + v >>= 7;
|
| + }
|
| + return 9;
|
| + }
|
| + n = 0;
|
| + do{
|
| + buf[n++] = (u8)((v & 0x7f) | 0x80);
|
| + v >>= 7;
|
| + }while( v!=0 );
|
| + buf[0] &= 0x7f;
|
| + assert( n<=9 );
|
| + for(i=0, j=n-1; j>=0; j--, i++){
|
| + p[i] = buf[j];
|
| + }
|
| + return n;
|
| +}
|
| +
|
| +static int sqlite3Fts5PutVarint(unsigned char *p, u64 v){
|
| + if( v<=0x7f ){
|
| + p[0] = v&0x7f;
|
| + return 1;
|
| + }
|
| + if( v<=0x3fff ){
|
| + p[0] = ((v>>7)&0x7f)|0x80;
|
| + p[1] = v&0x7f;
|
| + return 2;
|
| + }
|
| + return fts5PutVarint64(p,v);
|
| +}
|
| +
|
| +
|
| +static int sqlite3Fts5GetVarintLen(u32 iVal){
|
| + if( iVal<(1 << 7 ) ) return 1;
|
| + if( iVal<(1 << 14) ) return 2;
|
| + if( iVal<(1 << 21) ) return 3;
|
| + if( iVal<(1 << 28) ) return 4;
|
| + return 5;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** 2015 May 08
|
| +**
|
| +** The author disclaims copyright to this source code. In place of
|
| +** a legal notice, here is a blessing:
|
| +**
|
| +** May you do good and not evil.
|
| +** May you find forgiveness for yourself and forgive others.
|
| +** May you share freely, never taking more than you give.
|
| +**
|
| +******************************************************************************
|
| +**
|
| +** This is an SQLite virtual table module implementing direct access to an
|
| +** existing FTS5 index. The module may create several different types of
|
| +** tables:
|
| +**
|
| +** col:
|
| +** CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col));
|
| +**
|
| +** One row for each term/column combination. The value of $doc is set to
|
| +** the number of fts5 rows that contain at least one instance of term
|
| +** $term within column $col. Field $cnt is set to the total number of
|
| +** instances of term $term in column $col (in any row of the fts5 table).
|
| +**
|
| +** row:
|
| +** CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term));
|
| +**
|
| +** One row for each term in the database. The value of $doc is set to
|
| +** the number of fts5 rows that contain at least one instance of term
|
| +** $term. Field $cnt is set to the total number of instances of term
|
| +** $term in the database.
|
| +*/
|
| +
|
| +
|
| +/* #include "fts5Int.h" */
|
| +
|
| +
|
| +typedef struct Fts5VocabTable Fts5VocabTable;
|
| +typedef struct Fts5VocabCursor Fts5VocabCursor;
|
| +
|
| +struct Fts5VocabTable {
|
| + sqlite3_vtab base;
|
| + char *zFts5Tbl; /* Name of fts5 table */
|
| + char *zFts5Db; /* Db containing fts5 table */
|
| + sqlite3 *db; /* Database handle */
|
| + Fts5Global *pGlobal; /* FTS5 global object for this database */
|
| + int eType; /* FTS5_VOCAB_COL or ROW */
|
| +};
|
| +
|
| +struct Fts5VocabCursor {
|
| + sqlite3_vtab_cursor base;
|
| + sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */
|
| + Fts5Index *pIndex; /* Associated FTS5 index */
|
| +
|
| + int bEof; /* True if this cursor is at EOF */
|
| + Fts5IndexIter *pIter; /* Term/rowid iterator object */
|
| +
|
| + int nLeTerm; /* Size of zLeTerm in bytes */
|
| + char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */
|
| +
|
| + /* These are used by 'col' tables only */
|
| + Fts5Config *pConfig; /* Fts5 table configuration */
|
| + int iCol;
|
| + i64 *aCnt;
|
| + i64 *aDoc;
|
| +
|
| + /* Output values used by 'row' and 'col' tables */
|
| + i64 rowid; /* This table's current rowid value */
|
| + Fts5Buffer term; /* Current value of 'term' column */
|
| +};
|
| +
|
| +#define FTS5_VOCAB_COL 0
|
| +#define FTS5_VOCAB_ROW 1
|
| +
|
| +#define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt"
|
| +#define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt"
|
| +
|
| +/*
|
| +** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
|
| +*/
|
| +#define FTS5_VOCAB_TERM_EQ 0x01
|
| +#define FTS5_VOCAB_TERM_GE 0x02
|
| +#define FTS5_VOCAB_TERM_LE 0x04
|
| +
|
| +
|
| +/*
|
| +** Translate a string containing an fts5vocab table type to an
|
| +** FTS5_VOCAB_XXX constant. If successful, set *peType to the output
|
| +** value and return SQLITE_OK. Otherwise, set *pzErr to an error message
|
| +** and return SQLITE_ERROR.
|
| +*/
|
| +static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){
|
| + int rc = SQLITE_OK;
|
| + char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1);
|
| + if( rc==SQLITE_OK ){
|
| + sqlite3Fts5Dequote(zCopy);
|
| + if( sqlite3_stricmp(zCopy, "col")==0 ){
|
| + *peType = FTS5_VOCAB_COL;
|
| + }else
|
| +
|
| + if( sqlite3_stricmp(zCopy, "row")==0 ){
|
| + *peType = FTS5_VOCAB_ROW;
|
| + }else
|
| + {
|
| + *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy);
|
| + rc = SQLITE_ERROR;
|
| + }
|
| + sqlite3_free(zCopy);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** The xDisconnect() virtual table method.
|
| +*/
|
| +static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){
|
| + Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
|
| + sqlite3_free(pTab);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** The xDestroy() virtual table method.
|
| +*/
|
| +static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){
|
| + Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
|
| + sqlite3_free(pTab);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** This function is the implementation of both the xConnect and xCreate
|
| +** methods of the FTS3 virtual table.
|
| +**
|
| +** The argv[] array contains the following:
|
| +**
|
| +** argv[0] -> module name ("fts5vocab")
|
| +** argv[1] -> database name
|
| +** argv[2] -> table name
|
| +**
|
| +** then:
|
| +**
|
| +** argv[3] -> name of fts5 table
|
| +** argv[4] -> type of fts5vocab table
|
| +**
|
| +** or, for tables in the TEMP schema only.
|
| +**
|
| +** argv[3] -> name of fts5 tables database
|
| +** argv[4] -> name of fts5 table
|
| +** argv[5] -> type of fts5vocab table
|
| +*/
|
| +static int fts5VocabInitVtab(
|
| + sqlite3 *db, /* The SQLite database connection */
|
| + void *pAux, /* Pointer to Fts5Global object */
|
| + int argc, /* Number of elements in argv array */
|
| + const char * const *argv, /* xCreate/xConnect argument array */
|
| + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
|
| + char **pzErr /* Write any error message here */
|
| +){
|
| + const char *azSchema[] = {
|
| + "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")",
|
| + "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")"
|
| + };
|
| +
|
| + Fts5VocabTable *pRet = 0;
|
| + int rc = SQLITE_OK; /* Return code */
|
| + int bDb;
|
| +
|
| + bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0);
|
| +
|
| + if( argc!=5 && bDb==0 ){
|
| + *pzErr = sqlite3_mprintf("wrong number of vtable arguments");
|
| + rc = SQLITE_ERROR;
|
| + }else{
|
| + int nByte; /* Bytes of space to allocate */
|
| + const char *zDb = bDb ? argv[3] : argv[1];
|
| + const char *zTab = bDb ? argv[4] : argv[3];
|
| + const char *zType = bDb ? argv[5] : argv[4];
|
| + int nDb = (int)strlen(zDb)+1;
|
| + int nTab = (int)strlen(zTab)+1;
|
| + int eType = 0;
|
| +
|
| + rc = fts5VocabTableType(zType, pzErr, &eType);
|
| + if( rc==SQLITE_OK ){
|
| + assert( eType>=0 && eType<sizeof(azSchema)/sizeof(azSchema[0]) );
|
| + rc = sqlite3_declare_vtab(db, azSchema[eType]);
|
| + }
|
| +
|
| + nByte = sizeof(Fts5VocabTable) + nDb + nTab;
|
| + pRet = sqlite3Fts5MallocZero(&rc, nByte);
|
| + if( pRet ){
|
| + pRet->pGlobal = (Fts5Global*)pAux;
|
| + pRet->eType = eType;
|
| + pRet->db = db;
|
| + pRet->zFts5Tbl = (char*)&pRet[1];
|
| + pRet->zFts5Db = &pRet->zFts5Tbl[nTab];
|
| + memcpy(pRet->zFts5Tbl, zTab, nTab);
|
| + memcpy(pRet->zFts5Db, zDb, nDb);
|
| + sqlite3Fts5Dequote(pRet->zFts5Tbl);
|
| + sqlite3Fts5Dequote(pRet->zFts5Db);
|
| + }
|
| + }
|
| +
|
| + *ppVTab = (sqlite3_vtab*)pRet;
|
| + return rc;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** The xConnect() and xCreate() methods for the virtual table. All the
|
| +** work is done in function fts5VocabInitVtab().
|
| +*/
|
| +static int fts5VocabConnectMethod(
|
| + sqlite3 *db, /* Database connection */
|
| + void *pAux, /* Pointer to tokenizer hash table */
|
| + int argc, /* Number of elements in argv array */
|
| + const char * const *argv, /* xCreate/xConnect argument array */
|
| + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
|
| + char **pzErr /* OUT: sqlite3_malloc'd error message */
|
| +){
|
| + return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
|
| +}
|
| +static int fts5VocabCreateMethod(
|
| + sqlite3 *db, /* Database connection */
|
| + void *pAux, /* Pointer to tokenizer hash table */
|
| + int argc, /* Number of elements in argv array */
|
| + const char * const *argv, /* xCreate/xConnect argument array */
|
| + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
|
| + char **pzErr /* OUT: sqlite3_malloc'd error message */
|
| +){
|
| + return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
|
| +}
|
| +
|
| +/*
|
| +** Implementation of the xBestIndex method.
|
| +*/
|
| +static int fts5VocabBestIndexMethod(
|
| + sqlite3_vtab *pVTab,
|
| + sqlite3_index_info *pInfo
|
| +){
|
| + int i;
|
| + int iTermEq = -1;
|
| + int iTermGe = -1;
|
| + int iTermLe = -1;
|
| + int idxNum = 0;
|
| + int nArg = 0;
|
| +
|
| + for(i=0; i<pInfo->nConstraint; i++){
|
| + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
|
| + if( p->usable==0 ) continue;
|
| + if( p->iColumn==0 ){ /* term column */
|
| + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i;
|
| + if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i;
|
| + if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i;
|
| + if( p->op==SQLITE_INDEX_CONSTRAINT_GE ) iTermGe = i;
|
| + if( p->op==SQLITE_INDEX_CONSTRAINT_GT ) iTermGe = i;
|
| + }
|
| + }
|
| +
|
| + if( iTermEq>=0 ){
|
| + idxNum |= FTS5_VOCAB_TERM_EQ;
|
| + pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg;
|
| + pInfo->estimatedCost = 100;
|
| + }else{
|
| + pInfo->estimatedCost = 1000000;
|
| + if( iTermGe>=0 ){
|
| + idxNum |= FTS5_VOCAB_TERM_GE;
|
| + pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg;
|
| + pInfo->estimatedCost = pInfo->estimatedCost / 2;
|
| + }
|
| + if( iTermLe>=0 ){
|
| + idxNum |= FTS5_VOCAB_TERM_LE;
|
| + pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg;
|
| + pInfo->estimatedCost = pInfo->estimatedCost / 2;
|
| + }
|
| + }
|
| +
|
| + pInfo->idxNum = idxNum;
|
| +
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** Implementation of xOpen method.
|
| +*/
|
| +static int fts5VocabOpenMethod(
|
| + sqlite3_vtab *pVTab,
|
| + sqlite3_vtab_cursor **ppCsr
|
| +){
|
| + Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;
|
| + Fts5Index *pIndex = 0;
|
| + Fts5Config *pConfig = 0;
|
| + Fts5VocabCursor *pCsr = 0;
|
| + int rc = SQLITE_OK;
|
| + sqlite3_stmt *pStmt = 0;
|
| + char *zSql = 0;
|
| +
|
| + zSql = sqlite3Fts5Mprintf(&rc,
|
| + "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
|
| + pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl
|
| + );
|
| + if( zSql ){
|
| + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
|
| + }
|
| + sqlite3_free(zSql);
|
| + assert( rc==SQLITE_OK || pStmt==0 );
|
| + if( rc==SQLITE_ERROR ) rc = SQLITE_OK;
|
| +
|
| + if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
|
| + i64 iId = sqlite3_column_int64(pStmt, 0);
|
| + pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &pConfig);
|
| + }
|
| +
|
| + if( rc==SQLITE_OK && pIndex==0 ){
|
| + rc = sqlite3_finalize(pStmt);
|
| + pStmt = 0;
|
| + if( rc==SQLITE_OK ){
|
| + pVTab->zErrMsg = sqlite3_mprintf(
|
| + "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
|
| + );
|
| + rc = SQLITE_ERROR;
|
| + }
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + int nByte = pConfig->nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor);
|
| + pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
|
| + }
|
| +
|
| + if( pCsr ){
|
| + pCsr->pIndex = pIndex;
|
| + pCsr->pStmt = pStmt;
|
| + pCsr->pConfig = pConfig;
|
| + pCsr->aCnt = (i64*)&pCsr[1];
|
| + pCsr->aDoc = &pCsr->aCnt[pConfig->nCol];
|
| + }else{
|
| + sqlite3_finalize(pStmt);
|
| + }
|
| +
|
| + *ppCsr = (sqlite3_vtab_cursor*)pCsr;
|
| + return rc;
|
| +}
|
| +
|
| +static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){
|
| + pCsr->rowid = 0;
|
| + sqlite3Fts5IterClose(pCsr->pIter);
|
| + pCsr->pIter = 0;
|
| + sqlite3_free(pCsr->zLeTerm);
|
| + pCsr->nLeTerm = -1;
|
| + pCsr->zLeTerm = 0;
|
| +}
|
| +
|
| +/*
|
| +** Close the cursor. For additional information see the documentation
|
| +** on the xClose method of the virtual table interface.
|
| +*/
|
| +static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){
|
| + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
|
| + fts5VocabResetCursor(pCsr);
|
| + sqlite3Fts5BufferFree(&pCsr->term);
|
| + sqlite3_finalize(pCsr->pStmt);
|
| + sqlite3_free(pCsr);
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +
|
| +/*
|
| +** Advance the cursor to the next row in the table.
|
| +*/
|
| +static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
|
| + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
|
| + Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
|
| + int rc = SQLITE_OK;
|
| + int nCol = pCsr->pConfig->nCol;
|
| +
|
| + pCsr->rowid++;
|
| +
|
| + if( pTab->eType==FTS5_VOCAB_COL ){
|
| + for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
|
| + if( pCsr->aCnt[pCsr->iCol] ) break;
|
| + }
|
| + }
|
| +
|
| + if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){
|
| + if( sqlite3Fts5IterEof(pCsr->pIter) ){
|
| + pCsr->bEof = 1;
|
| + }else{
|
| + const char *zTerm;
|
| + int nTerm;
|
| +
|
| + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
|
| + if( pCsr->nLeTerm>=0 ){
|
| + int nCmp = MIN(nTerm, pCsr->nLeTerm);
|
| + int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
|
| + if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
|
| + pCsr->bEof = 1;
|
| + return SQLITE_OK;
|
| + }
|
| + }
|
| +
|
| + sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
|
| + memset(pCsr->aCnt, 0, nCol * sizeof(i64));
|
| + memset(pCsr->aDoc, 0, nCol * sizeof(i64));
|
| + pCsr->iCol = 0;
|
| +
|
| + assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
|
| + while( rc==SQLITE_OK ){
|
| + i64 dummy;
|
| + const u8 *pPos; int nPos; /* Position list */
|
| + i64 iPos = 0; /* 64-bit position read from poslist */
|
| + int iOff = 0; /* Current offset within position list */
|
| +
|
| + rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy);
|
| + if( rc==SQLITE_OK ){
|
| + if( pTab->eType==FTS5_VOCAB_ROW ){
|
| + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
|
| + pCsr->aCnt[0]++;
|
| + }
|
| + pCsr->aDoc[0]++;
|
| + }else{
|
| + int iCol = -1;
|
| + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
|
| + int ii = FTS5_POS2COLUMN(iPos);
|
| + pCsr->aCnt[ii]++;
|
| + if( iCol!=ii ){
|
| + pCsr->aDoc[ii]++;
|
| + iCol = ii;
|
| + }
|
| + }
|
| + }
|
| + rc = sqlite3Fts5IterNextScan(pCsr->pIter);
|
| + }
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
|
| + if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){
|
| + break;
|
| + }
|
| + if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
|
| + }
|
| + }
|
| + }
|
| + }
|
| +
|
| + if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
|
| + while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++;
|
| + assert( pCsr->iCol<pCsr->pConfig->nCol );
|
| + }
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** This is the xFilter implementation for the virtual table.
|
| +*/
|
| +static int fts5VocabFilterMethod(
|
| + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
|
| + int idxNum, /* Strategy index */
|
| + const char *idxStr, /* Unused */
|
| + int nVal, /* Number of elements in apVal */
|
| + sqlite3_value **apVal /* Arguments for the indexing scheme */
|
| +){
|
| + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
|
| + int rc = SQLITE_OK;
|
| +
|
| + int iVal = 0;
|
| + int f = FTS5INDEX_QUERY_SCAN;
|
| + const char *zTerm = 0;
|
| + int nTerm = 0;
|
| +
|
| + sqlite3_value *pEq = 0;
|
| + sqlite3_value *pGe = 0;
|
| + sqlite3_value *pLe = 0;
|
| +
|
| + fts5VocabResetCursor(pCsr);
|
| + if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
|
| + if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
|
| + if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
|
| +
|
| + if( pEq ){
|
| + zTerm = (const char *)sqlite3_value_text(pEq);
|
| + nTerm = sqlite3_value_bytes(pEq);
|
| + f = 0;
|
| + }else{
|
| + if( pGe ){
|
| + zTerm = (const char *)sqlite3_value_text(pGe);
|
| + nTerm = sqlite3_value_bytes(pGe);
|
| + }
|
| + if( pLe ){
|
| + const char *zCopy = (const char *)sqlite3_value_text(pLe);
|
| + pCsr->nLeTerm = sqlite3_value_bytes(pLe);
|
| + pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1);
|
| + if( pCsr->zLeTerm==0 ){
|
| + rc = SQLITE_NOMEM;
|
| + }else{
|
| + memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1);
|
| + }
|
| + }
|
| + }
|
| +
|
| +
|
| + if( rc==SQLITE_OK ){
|
| + rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
|
| + }
|
| + if( rc==SQLITE_OK ){
|
| + rc = fts5VocabNextMethod(pCursor);
|
| + }
|
| +
|
| + return rc;
|
| +}
|
| +
|
| +/*
|
| +** This is the xEof method of the virtual table. SQLite calls this
|
| +** routine to find out if it has reached the end of a result set.
|
| +*/
|
| +static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){
|
| + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
|
| + return pCsr->bEof;
|
| +}
|
| +
|
| +static int fts5VocabColumnMethod(
|
| + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
|
| + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
|
| + int iCol /* Index of column to read value from */
|
| +){
|
| + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
|
| +
|
| + if( iCol==0 ){
|
| + sqlite3_result_text(
|
| + pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
|
| + );
|
| + }
|
| + else if( ((Fts5VocabTable*)(pCursor->pVtab))->eType==FTS5_VOCAB_COL ){
|
| + assert( iCol==1 || iCol==2 || iCol==3 );
|
| + if( iCol==1 ){
|
| + const char *z = pCsr->pConfig->azCol[pCsr->iCol];
|
| + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
|
| + }else if( iCol==2 ){
|
| + sqlite3_result_int64(pCtx, pCsr->aDoc[pCsr->iCol]);
|
| + }else{
|
| + sqlite3_result_int64(pCtx, pCsr->aCnt[pCsr->iCol]);
|
| + }
|
| + }else{
|
| + assert( iCol==1 || iCol==2 );
|
| + if( iCol==1 ){
|
| + sqlite3_result_int64(pCtx, pCsr->aDoc[0]);
|
| + }else{
|
| + sqlite3_result_int64(pCtx, pCsr->aCnt[0]);
|
| + }
|
| + }
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +/*
|
| +** This is the xRowid method. The SQLite core calls this routine to
|
| +** retrieve the rowid for the current row of the result set. The
|
| +** rowid should be written to *pRowid.
|
| +*/
|
| +static int fts5VocabRowidMethod(
|
| + sqlite3_vtab_cursor *pCursor,
|
| + sqlite_int64 *pRowid
|
| +){
|
| + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
|
| + *pRowid = pCsr->rowid;
|
| + return SQLITE_OK;
|
| +}
|
| +
|
| +static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
|
| + static const sqlite3_module fts5Vocab = {
|
| + /* iVersion */ 2,
|
| + /* xCreate */ fts5VocabCreateMethod,
|
| + /* xConnect */ fts5VocabConnectMethod,
|
| + /* xBestIndex */ fts5VocabBestIndexMethod,
|
| + /* xDisconnect */ fts5VocabDisconnectMethod,
|
| + /* xDestroy */ fts5VocabDestroyMethod,
|
| + /* xOpen */ fts5VocabOpenMethod,
|
| + /* xClose */ fts5VocabCloseMethod,
|
| + /* xFilter */ fts5VocabFilterMethod,
|
| + /* xNext */ fts5VocabNextMethod,
|
| + /* xEof */ fts5VocabEofMethod,
|
| + /* xColumn */ fts5VocabColumnMethod,
|
| + /* xRowid */ fts5VocabRowidMethod,
|
| + /* xUpdate */ 0,
|
| + /* xBegin */ 0,
|
| + /* xSync */ 0,
|
| + /* xCommit */ 0,
|
| + /* xRollback */ 0,
|
| + /* xFindFunction */ 0,
|
| + /* xRename */ 0,
|
| + /* xSavepoint */ 0,
|
| + /* xRelease */ 0,
|
| + /* xRollbackTo */ 0,
|
| + };
|
| + void *p = (void*)pGlobal;
|
| +
|
| + return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
|
| +}
|
| +
|
| +
|
| +
|
| +
|
| +
|
| +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */
|
| +
|
| +/************** End of fts5.c ************************************************/
|
|
|
Chromium Code Reviews
Issue 1636873003:
Try for backport (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@zzsql_import3_10_2_websql_backport