Make sure my recent changes are represented in the patches....

third_party/sqlite/misc.patch

 Index: Makefile.linux-gcc
 ===================================================================
 --- Makefile.linux-gcc  2009-09-03 13:32:06.000000000 -0700
 +++ Makefile.linux-gcc  2009-07-01 12:08:39.000000000 -0700
 @@ -14,7 +14,7 @@
  #### The toplevel directory of the source tree.  This is the directory
  #    that contains this "Makefile.in" and the "configure.in" script.
  #
 -TOP = ../sqlite
 +TOP = ..
@@ skipping 77 matching lines @@
      break;
    }
    return rc;
 -
 - err:
 -  sqlite3_finalize(s);
 -  return rc;
  }
  
  /* Like sql_step_statement(), but convert SQLITE_DONE to SQLITE_OK.
-Index: ext/fts2/fts2.c
-===================================================================
---- ext/fts2/fts2.c     2009-09-04 13:37:41.000000000 -0700
-+++ ext/fts2/fts2.c     2009-09-14 18:17:02.000000000 -0700
-@@ -37,6 +37,20 @@
- ** This is an SQLite module implementing full-text search.
- */
- 
-+/* TODO(shess): To make it easier to spot changes without groveling
-+** through changelogs, I've defined GEARS_FTS2_CHANGES to call them
-+** out, and I will document them here.  On imports, these changes
-+** should be reviewed to make sure they are still present, or are
-+** dropped as appropriate.
-+**
-+** SQLite core adds the custom function fts2_tokenizer() to be used
-+** for defining new tokenizers.  The second parameter is a vtable
-+** pointer encoded as a blob.  Obviously this cannot be exposed to
-+** Gears callers for security reasons.  It could be suppressed in the
-+** authorizer, but for now I have simply commented the definition out.
-+*/
-+#define GEARS_FTS2_CHANGES 1
-+
- /*
- ** The code in this file is only compiled if:
- **
-@@ -335,6 +349,16 @@
- # define TRACE(A)
- #endif
- 
-+#if 0
-+/* Useful to set breakpoints.  See main.c sqlite3Corrupt(). */
-+static int fts2Corrupt(void){
-+  return SQLITE_CORRUPT;
-+}
-+# define SQLITE_CORRUPT_BKPT fts2Corrupt()
-+#else
-+# define SQLITE_CORRUPT_BKPT SQLITE_CORRUPT
-+#endif
-+
- /* It is not safe to call isspace(), tolower(), or isalnum() on
- ** hi-bit-set characters.  This is the same solution used in the
- ** tokenizer.
-@@ -1814,7 +1838,7 @@
-   /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
-   /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
-   /* SEGDIR_SELECT_LEVEL */
--  "select start_block, leaves_end_block, root from %_segdir "
-+  "select start_block, leaves_end_block, root, idx from %_segdir "
-   " where level = ? order by idx",
-   /* SEGDIR_SPAN */
-   "select min(start_block), max(end_block) from %_segdir "
-@@ -3421,8 +3445,11 @@
-       c->eof = 0;
-       return SQLITE_OK;
-     }
--    /* an error occurred; abort */
--    return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
-+
-+    /* Corrupt if the index refers to missing document. */
-+    if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT;
-+
-+    return rc;
-   }
- }
- 
-@@ -3544,6 +3571,7 @@
-   int firstIndex = pQuery->nTerms;
-   int iCol;
-   int nTerm = 1;
-+  int iEndLast = -1;
-   
-   int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor);
-   if( rc!=SQLITE_OK ) return rc;
-@@ -3568,6 +3596,20 @@
-       pQuery->nextIsOr = 1;
-       continue;
-     }
-+
-+    /*
-+     * The ICU tokenizer considers '*' a break character, so the code below
-+     * sets isPrefix correctly, but since that code doesn't eat the '*', the
-+     * ICU tokenizer returns it as the next token.  So eat it here until a
-+     * better solution presents itself.
-+     */
-+    if( pQuery->nTerms>0 && nToken==1 && pSegment[iBegin]=='*' &&
-+        iEndLast==iBegin){
-+      pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1;
-+      continue;
-+    }
-+    iEndLast = iEnd;
-+    
-     queryAdd(pQuery, pToken, nToken);
-     if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){
-       pQuery->pTerms[pQuery->nTerms-1].isNot = 1;
-@@ -5077,6 +5119,9 @@
- ** the leaf data was entirely contained in the root), or from the
- ** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
- */
-+/* TODO(shess): Figure out a means of indicating how many leaves are
-+** expected, for purposes of detecting corruption.
-+*/
- static int leavesReaderInit(fulltext_vtab *v,
-                             int idx,
-                             sqlite_int64 iStartBlockid,
-@@ -5088,6 +5133,10 @@
- 
-   dataBufferInit(&pReader->rootData, 0);
-   if( iStartBlockid==0 ){
-+    /* Corrupt if this can't be a leaf node. */
-+    if( pRootData==NULL || nRootData<1 || pRootData[0]!='\0' ){
-+      return SQLITE_CORRUPT_BKPT;
-+    }
-     /* Entire leaf level fit in root data. */
-     dataBufferReplace(&pReader->rootData, pRootData, nRootData);
-     leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
-@@ -5098,22 +5147,48 @@
-     if( rc!=SQLITE_OK ) return rc;
- 
-     rc = sqlite3_bind_int64(s, 1, iStartBlockid);
--    if( rc!=SQLITE_OK ) return rc;
-+    if( rc!=SQLITE_OK ) goto err;
- 
-     rc = sqlite3_bind_int64(s, 2, iEndBlockid);
--    if( rc!=SQLITE_OK ) return rc;
-+    if( rc!=SQLITE_OK ) goto err;
- 
-     rc = sqlite3_step(s);
-+
-+    /* Corrupt if interior node referenced missing leaf node. */
-     if( rc==SQLITE_DONE ){
--      pReader->eof = 1;
--      return SQLITE_OK;
-+      rc = SQLITE_CORRUPT_BKPT;
-+      goto err;
-+    }
-+
-+    if( rc!=SQLITE_ROW ) goto err;
-+    rc = SQLITE_OK;
-+
-+    /* Corrupt if leaf data isn't a blob. */
-+    if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){
-+      rc = SQLITE_CORRUPT_BKPT;
-+    }else{
-+      const char *pLeafData = sqlite3_column_blob(s, 0);
-+      int nLeafData = sqlite3_column_bytes(s, 0);
-+
-+      /* Corrupt if this can't be a leaf node. */
-+      if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
-+        rc = SQLITE_CORRUPT_BKPT;
-+      }else{
-+        leafReaderInit(pLeafData, nLeafData, &pReader->leafReader);
-+      }
-+    }
-+
-+ err:
-+    if( rc!=SQLITE_OK ){
-+      if( idx==-1 ){
-+        sqlite3_finalize(s);
-+      }else{
-+        sqlite3_reset(s);
-+      }
-+      return rc;
-     }
--    if( rc!=SQLITE_ROW ) return rc;
- 
-     pReader->pStmt = s;
--    leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
--                   sqlite3_column_bytes(pReader->pStmt, 0),
--                   &pReader->leafReader);
-   }
-   return SQLITE_OK;
- }
-@@ -5136,10 +5211,22 @@
-       pReader->eof = 1;
-       return rc==SQLITE_DONE ? SQLITE_OK : rc;
-     }
--    leafReaderDestroy(&pReader->leafReader);
--    leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
--                   sqlite3_column_bytes(pReader->pStmt, 0),
--                   &pReader->leafReader);
-+
-+    /* Corrupt if leaf data isn't a blob. */
-+    if( sqlite3_column_type(pReader->pStmt, 0)!=SQLITE_BLOB ){
-+      return SQLITE_CORRUPT_BKPT;
-+    }else{
-+      const char *pLeafData = sqlite3_column_blob(pReader->pStmt, 0);
-+      int nLeafData = sqlite3_column_bytes(pReader->pStmt, 0);
-+
-+      /* Corrupt if this can't be a leaf node. */
-+      if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
-+        return SQLITE_CORRUPT_BKPT;
-+      }
-+
-+      leafReaderDestroy(&pReader->leafReader);
-+      leafReaderInit(pLeafData, nLeafData, &pReader->leafReader);
-+    }
-   }
-   return SQLITE_OK;
- }
-@@ -5200,8 +5287,19 @@
-     sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
-     const char *pRootData = sqlite3_column_blob(s, 2);
-     int nRootData = sqlite3_column_bytes(s, 2);
-+    sqlite_int64 iIndex = sqlite3_column_int64(s, 3);
-+
-+    /* Corrupt if we get back different types than we stored. */
-+    /* Also corrupt if the index is not sequential starting at 0. */
-+    if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
-+        sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-+        sqlite3_column_type(s, 2)!=SQLITE_BLOB ||
-+        i!=iIndex ||
-+        i>=MERGE_COUNT ){
-+      rc = SQLITE_CORRUPT_BKPT;
-+      break;
-+    }
- 
--    assert( i<MERGE_COUNT );
-     rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
-                           &pReaders[i]);
-     if( rc!=SQLITE_OK ) break;
-@@ -5212,6 +5310,7 @@
-     while( i-->0 ){
-       leavesReaderDestroy(&pReaders[i]);
-     }
-+    sqlite3_reset(s);          /* So we don't leave a lock. */
-     return rc;
-   }
- 
-@@ -5295,10 +5394,14 @@
-   memset(&lrs, '\0', sizeof(lrs));
-   rc = leavesReadersInit(v, iLevel, lrs, &i);
-   if( rc!=SQLITE_OK ) return rc;
--  assert( i==MERGE_COUNT );
- 
-   leafWriterInit(iLevel+1, idx, &writer);
- 
-+  if( i!=MERGE_COUNT ){
-+    rc = SQLITE_CORRUPT_BKPT;
-+    goto err;
-+  }
-+
-   /* Since leavesReaderReorder() pushes readers at eof to the end,
-   ** when the first reader is empty, all will be empty.
-   */
-@@ -5588,11 +5691,27 @@
-   if( rc!=SQLITE_OK ) return rc;
- 
-   rc = sqlite3_step(s);
--  if( rc==SQLITE_DONE ) return SQLITE_ERROR;
-+  /* Corrupt if interior node references missing child node. */
-+  if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT;
-   if( rc!=SQLITE_ROW ) return rc;
- 
--  getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
--                        pTerm, nTerm, isPrefix, piStartChild, piEndChild);
-+  /* Corrupt if child node isn't a blob. */
-+  if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){
-+    sqlite3_reset(s);         /* So we don't leave a lock. */
-+    return SQLITE_CORRUPT_BKPT;
-+  }else{
-+    const char *pData = sqlite3_column_blob(s, 0);
-+    int nData = sqlite3_column_bytes(s, 0);
-+
-+    /* Corrupt if child is not a valid interior node. */
-+    if( pData==NULL || nData<1 || pData[0]=='\0' ){
-+      sqlite3_reset(s);         /* So we don't leave a lock. */
-+      return SQLITE_CORRUPT_BKPT;
-+    }
-+
-+    getChildrenContaining(pData, nData, pTerm, nTerm,
-+                          isPrefix, piStartChild, piEndChild);
-+  }
- 
-   /* We expect only one row.  We must execute another sqlite3_step()
-    * to complete the iteration; otherwise the table will remain
-@@ -5675,7 +5794,8 @@
-   DataBuffer result;
-   int rc;
- 
--  assert( nData>1 );
-+  /* Corrupt if segment root can't be valid. */
-+  if( pData==NULL || nData<1 ) return SQLITE_CORRUPT_BKPT;
- 
-   /* This code should never be called with buffered updates. */
-   assert( v->nPendingData<0 );
-@@ -5729,6 +5849,14 @@
-     const char *pData = sqlite3_column_blob(s, 2);
-     const int nData = sqlite3_column_bytes(s, 2);
-     const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
-+
-+    /* Corrupt if we get back different types than we stored. */
-+    if( sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-+        sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
-+      rc = SQLITE_CORRUPT_BKPT;
-+      goto err;
-+    }
-+
-     rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
-                      &doclist);
-     if( rc!=SQLITE_OK ) goto err;
-@@ -5748,6 +5876,7 @@
-   }
- 
-  err:
-+  sqlite3_reset(s);         /* So we don't leave a lock. */
-   dataBufferDestroy(&doclist);
-   return rc;
- }
-@@ -6240,6 +6369,14 @@
-       const char *pRootData = sqlite3_column_blob(s, 2);
-       int nRootData = sqlite3_column_bytes(s, 2);
- 
-+      /* Corrupt if we get back different types than we stored. */
-+      if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
-+          sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-+          sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
-+        rc = SQLITE_CORRUPT_BKPT;
-+        break;
-+      }
-+
-       assert( i<nReaders );
-       rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
-                             &readers[i].reader);
-@@ -6253,6 +6390,8 @@
-     if( rc==SQLITE_DONE ){
-       assert( i==nReaders );
-       rc = optimizeInternal(v, readers, nReaders, &writer);
-+    }else{
-+      sqlite3_reset(s);      /* So we don't leave a lock. */
-     }
- 
-     while( i-- > 0 ){
-@@ -6316,9 +6455,18 @@
-   const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
-   const char *pRootData = sqlite3_column_blob(s, 2);
-   const int nRootData = sqlite3_column_bytes(s, 2);
-+  int rc;
-   LeavesReader reader;
--  int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
--                            pRootData, nRootData, &reader);
-+
-+  /* Corrupt if we get back different types than we stored. */
-+  if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
-+      sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-+      sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
-+    return SQLITE_CORRUPT_BKPT;
-+  }
-+
-+  rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
-+                        pRootData, nRootData, &reader);
-   if( rc!=SQLITE_OK ) return rc;
- 
-   while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
-@@ -6822,7 +6970,11 @@
-   ** module with sqlite.
-   */
-   if( SQLITE_OK==rc 
-+#if GEARS_FTS2_CHANGES && !SQLITE_TEST
-+      /* fts2_tokenizer() disabled for security reasons. */
-+#else
-    && SQLITE_OK==(rc = sqlite3Fts2InitHashTable(db, pHash, "fts2_tokenizer"))
-+#endif
-    && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
-    && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
-    && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
-Index: ext/fts2/fts2_icu.c
-===================================================================
---- ext/fts2/fts2_icu.c 2009-09-03 13:32:06.000000000 -0700
-+++ ext/fts2/fts2_icu.c 2009-09-14 18:17:16.000000000 -0700
-@@ -198,7 +198,7 @@
- 
-     while( iStart<iEnd ){
-       int iWhite = iStart;
--      U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
-+      U16_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
-       if( u_isspace(c) ){
-         iStart = iWhite;
-       }else{
-Index: ext/fts2/fts2_tokenizer.c
-===================================================================
---- ext/fts2/fts2_tokenizer.c   2009-09-03 13:32:06.000000000 -0700
-+++ ext/fts2/fts2_tokenizer.c   2009-09-14 18:17:24.000000000 -0700
-@@ -33,6 +33,7 @@
- #include "fts2_hash.h"
- #include "fts2_tokenizer.h"
- #include <assert.h>
-+#include <stddef.h>
- 
- /*
- ** Implementation of the SQL scalar function for accessing the underlying 
 Index: ext/icu/icu.c
 ===================================================================
 --- ext/icu/icu.c       2009-09-03 13:32:06.000000000 -0700
 +++ ext/icu/icu.c       2009-07-01 12:08:37.000000000 -0700
 @@ -38,6 +38,11 @@
  
  #include <assert.h>
  
 +// TODO(evanm): this is cut'n'pasted from fts2.c.  Why is it necessary?
 +#if !defined(SQLITE_CORE)
@@ skipping 1088 matching lines @@
 +  if {$sqlite_open_file_count>0} {
 +    puts "$tail did not close all files: $sqlite_open_file_count"
 +    incr nErr
 +    lappend ::failList $tail
 +  }
 +}
 +source $testdir/misuse.test
 +
 +set sqlite_open_file_count 0
 +really_finish_test