OLD | NEW |
(Empty) | |
| 1 /* |
| 2 ** 2016-12-28 |
| 3 ** |
| 4 ** The author disclaims copyright to this source code. In place of |
| 5 ** a legal notice, here is a blessing: |
| 6 ** |
| 7 ** May you do good and not evil. |
| 8 ** May you find forgiveness for yourself and forgive others. |
| 9 ** May you share freely, never taking more than you give. |
| 10 ** |
| 11 ************************************************************************* |
| 12 ** |
| 13 ** This file implements "key-value" performance test for SQLite. The |
| 14 ** purpose is to compare the speed of SQLite for accessing large BLOBs |
| 15 ** versus reading those same BLOB values out of individual files in the |
| 16 ** filesystem. |
| 17 ** |
| 18 ** Run "kvtest" with no arguments for on-line help, or see comments below. |
| 19 ** |
| 20 ** HOW TO COMPILE: |
| 21 ** |
| 22 ** (1) Gather this source file and a recent SQLite3 amalgamation with its |
| 23 ** header into the working directory. You should have: |
| 24 ** |
| 25 ** kvtest.c >--- this file |
| 26 ** sqlite3.c \___ SQLite |
| 27 ** sqlite3.h / amlagamation & header |
| 28 ** |
| 29 ** (2) Run you compiler against the two C source code files. |
| 30 ** |
| 31 ** (a) On linux or mac: |
| 32 ** |
| 33 ** OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION" |
| 34 ** gcc -Os -I. $OPTS kvtest.c sqlite3.c -o kvtest |
| 35 ** |
| 36 ** The $OPTS options can be omitted. The $OPTS merely omit |
| 37 ** the need to link against -ldl and -lpthread, or whatever |
| 38 ** the equivalent libraries are called on your system. |
| 39 ** |
| 40 ** (b) Windows with MSVC: |
| 41 ** |
| 42 ** cl -I. kvtest.c sqlite3.c |
| 43 ** |
| 44 ** USAGE: |
| 45 ** |
| 46 ** (1) Create a test database by running "kvtest init" with appropriate |
| 47 ** options. See the help message for available options. |
| 48 ** |
| 49 ** (2) Construct the corresponding pile-of-files database on disk using |
| 50 ** the "kvtest export" command. |
| 51 ** |
| 52 ** (3) Run tests using "kvtest run" against either the SQLite database or |
| 53 ** the pile-of-files database and with appropriate options. |
| 54 ** |
| 55 ** For example: |
| 56 ** |
| 57 ** ./kvtest init x1.db --count 100000 --size 10000 |
| 58 ** mkdir x1 |
| 59 ** ./kvtest export x1.db x1 |
| 60 ** ./kvtest run x1.db --count 10000 --max-id 1000000 |
| 61 ** ./kvtest run x1 --count 10000 --max-id 1000000 |
| 62 */ |
| 63 static const char zHelp[] = |
| 64 "Usage: kvtest COMMAND ARGS...\n" |
| 65 "\n" |
| 66 " kvtest init DBFILE --count N --size M --pagesize X\n" |
| 67 "\n" |
| 68 " Generate a new test database file named DBFILE containing N\n" |
| 69 " BLOBs each of size M bytes. The page size of the new database\n" |
| 70 " file will be X. Additional options:\n" |
| 71 "\n" |
| 72 " --variance V Randomly vary M by plus or minus V\n" |
| 73 "\n" |
| 74 " kvtest export DBFILE DIRECTORY\n" |
| 75 "\n" |
| 76 " Export all the blobs in the kv table of DBFILE into separate\n" |
| 77 " files in DIRECTORY.\n" |
| 78 "\n" |
| 79 " kvtest stat DBFILE\n" |
| 80 "\n" |
| 81 " Display summary information about DBFILE\n" |
| 82 "\n" |
| 83 " kvtest run DBFILE [options]\n" |
| 84 "\n" |
| 85 " Run a performance test. DBFILE can be either the name of a\n" |
| 86 " database or a directory containing sample files. Options:\n" |
| 87 "\n" |
| 88 " --asc Read blobs in ascending order\n" |
| 89 " --blob-api Use the BLOB API\n" |
| 90 " --cache-size N Database cache size\n" |
| 91 " --count N Read N blobs\n" |
| 92 " --desc Read blobs in descending order\n" |
| 93 " --max-id N Maximum blob key to use\n" |
| 94 " --mmap N Mmap as much as N bytes of DBFILE\n" |
| 95 " --jmode MODE Set MODE journal mode prior to starting\n" |
| 96 " --random Read blobs in a random order\n" |
| 97 " --start N Start reading with this blob key\n" |
| 98 " --stats Output operating stats before exiting\n" |
| 99 ; |
| 100 |
| 101 /* Reference resources used */ |
| 102 #include <stdio.h> |
| 103 #include <stdlib.h> |
| 104 #include <sys/types.h> |
| 105 #include <sys/stat.h> |
| 106 #include <assert.h> |
| 107 #include <string.h> |
| 108 #include "sqlite3.h" |
| 109 |
| 110 #ifndef _WIN32 |
| 111 # include <unistd.h> |
| 112 #else |
| 113 /* Provide Windows equivalent for the needed parts of unistd.h */ |
| 114 # include <io.h> |
| 115 # define R_OK 2 |
| 116 # define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) |
| 117 # define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) |
| 118 # define access _access |
| 119 #endif |
| 120 |
| 121 |
| 122 /* |
| 123 ** Show thqe help text and quit. |
| 124 */ |
| 125 static void showHelp(void){ |
| 126 fprintf(stdout, "%s", zHelp); |
| 127 exit(1); |
| 128 } |
| 129 |
| 130 /* |
| 131 ** Show an error message an quit. |
| 132 */ |
| 133 static void fatalError(const char *zFormat, ...){ |
| 134 va_list ap; |
| 135 fprintf(stdout, "ERROR: "); |
| 136 va_start(ap, zFormat); |
| 137 vfprintf(stdout, zFormat, ap); |
| 138 va_end(ap); |
| 139 fprintf(stdout, "\n"); |
| 140 exit(1); |
| 141 } |
| 142 |
| 143 /* |
| 144 ** Return the value of a hexadecimal digit. Return -1 if the input |
| 145 ** is not a hex digit. |
| 146 */ |
| 147 static int hexDigitValue(char c){ |
| 148 if( c>='0' && c<='9' ) return c - '0'; |
| 149 if( c>='a' && c<='f' ) return c - 'a' + 10; |
| 150 if( c>='A' && c<='F' ) return c - 'A' + 10; |
| 151 return -1; |
| 152 } |
| 153 |
| 154 /* |
| 155 ** Interpret zArg as an integer value, possibly with suffixes. |
| 156 */ |
| 157 static int integerValue(const char *zArg){ |
| 158 int v = 0; |
| 159 static const struct { char *zSuffix; int iMult; } aMult[] = { |
| 160 { "KiB", 1024 }, |
| 161 { "MiB", 1024*1024 }, |
| 162 { "GiB", 1024*1024*1024 }, |
| 163 { "KB", 1000 }, |
| 164 { "MB", 1000000 }, |
| 165 { "GB", 1000000000 }, |
| 166 { "K", 1000 }, |
| 167 { "M", 1000000 }, |
| 168 { "G", 1000000000 }, |
| 169 }; |
| 170 int i; |
| 171 int isNeg = 0; |
| 172 if( zArg[0]=='-' ){ |
| 173 isNeg = 1; |
| 174 zArg++; |
| 175 }else if( zArg[0]=='+' ){ |
| 176 zArg++; |
| 177 } |
| 178 if( zArg[0]=='0' && zArg[1]=='x' ){ |
| 179 int x; |
| 180 zArg += 2; |
| 181 while( (x = hexDigitValue(zArg[0]))>=0 ){ |
| 182 v = (v<<4) + x; |
| 183 zArg++; |
| 184 } |
| 185 }else{ |
| 186 while( zArg[0]>='0' && zArg[0]<='9' ){ |
| 187 v = v*10 + zArg[0] - '0'; |
| 188 zArg++; |
| 189 } |
| 190 } |
| 191 for(i=0; i<sizeof(aMult)/sizeof(aMult[0]); i++){ |
| 192 if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){ |
| 193 v *= aMult[i].iMult; |
| 194 break; |
| 195 } |
| 196 } |
| 197 return isNeg? -v : v; |
| 198 } |
| 199 |
| 200 |
| 201 /* |
| 202 ** Check the filesystem object zPath. Determine what it is: |
| 203 ** |
| 204 ** PATH_DIR A directory |
| 205 ** PATH_DB An SQLite database |
| 206 ** PATH_NEXIST Does not exist |
| 207 ** PATH_OTHER Something else |
| 208 */ |
| 209 #define PATH_DIR 1 |
| 210 #define PATH_DB 2 |
| 211 #define PATH_NEXIST 0 |
| 212 #define PATH_OTHER 99 |
| 213 static int pathType(const char *zPath){ |
| 214 struct stat x; |
| 215 int rc; |
| 216 if( access(zPath,R_OK) ) return PATH_NEXIST; |
| 217 memset(&x, 0, sizeof(x)); |
| 218 rc = stat(zPath, &x); |
| 219 if( rc<0 ) return PATH_OTHER; |
| 220 if( S_ISDIR(x.st_mode) ) return PATH_DIR; |
| 221 if( (x.st_size%512)==0 ) return PATH_DB; |
| 222 return PATH_OTHER; |
| 223 } |
| 224 |
| 225 /* |
| 226 ** Return the size of a file in bytes. Or return -1 if the |
| 227 ** named object is not a regular file or does not exist. |
| 228 */ |
| 229 static sqlite3_int64 fileSize(const char *zPath){ |
| 230 struct stat x; |
| 231 int rc; |
| 232 memset(&x, 0, sizeof(x)); |
| 233 rc = stat(zPath, &x); |
| 234 if( rc<0 ) return -1; |
| 235 if( !S_ISREG(x.st_mode) ) return -1; |
| 236 return x.st_size; |
| 237 } |
| 238 |
| 239 /* |
| 240 ** A Pseudo-random number generator with a fixed seed. Use this so |
| 241 ** that the same sequence of "random" numbers are generated on each |
| 242 ** run, for repeatability. |
| 243 */ |
| 244 static unsigned int randInt(void){ |
| 245 static unsigned int x = 0x333a13cd; |
| 246 static unsigned int y = 0xecb2adea; |
| 247 x = (x>>1) ^ ((1+~(x&1)) & 0xd0000001); |
| 248 y = y*1103515245 + 12345; |
| 249 return x^y; |
| 250 } |
| 251 |
| 252 /* |
| 253 ** Do database initialization. |
| 254 */ |
| 255 static int initMain(int argc, char **argv){ |
| 256 char *zDb; |
| 257 int i, rc; |
| 258 int nCount = 1000; |
| 259 int sz = 10000; |
| 260 int iVariance = 0; |
| 261 int pgsz = 4096; |
| 262 sqlite3 *db; |
| 263 char *zSql; |
| 264 char *zErrMsg = 0; |
| 265 |
| 266 assert( strcmp(argv[1],"init")==0 ); |
| 267 assert( argc>=3 ); |
| 268 zDb = argv[2]; |
| 269 for(i=3; i<argc; i++){ |
| 270 char *z = argv[i]; |
| 271 if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z); |
| 272 if( z[1]=='-' ) z++; |
| 273 if( strcmp(z, "-count")==0 ){ |
| 274 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); |
| 275 nCount = integerValue(argv[++i]); |
| 276 if( nCount<1 ) fatalError("the --count must be positive"); |
| 277 continue; |
| 278 } |
| 279 if( strcmp(z, "-size")==0 ){ |
| 280 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); |
| 281 sz = integerValue(argv[++i]); |
| 282 if( sz<1 ) fatalError("the --size must be positive"); |
| 283 continue; |
| 284 } |
| 285 if( strcmp(z, "-variance")==0 ){ |
| 286 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); |
| 287 iVariance = integerValue(argv[++i]); |
| 288 continue; |
| 289 } |
| 290 if( strcmp(z, "-pagesize")==0 ){ |
| 291 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); |
| 292 pgsz = integerValue(argv[++i]); |
| 293 if( pgsz<512 || pgsz>65536 || ((pgsz-1)&pgsz)!=0 ){ |
| 294 fatalError("the --pagesize must be power of 2 between 512 and 65536"); |
| 295 } |
| 296 continue; |
| 297 } |
| 298 fatalError("unknown option: \"%s\"", argv[i]); |
| 299 } |
| 300 rc = sqlite3_open(zDb, &db); |
| 301 if( rc ){ |
| 302 fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db)); |
| 303 } |
| 304 zSql = sqlite3_mprintf( |
| 305 "DROP TABLE IF EXISTS kv;\n" |
| 306 "PRAGMA page_size=%d;\n" |
| 307 "VACUUM;\n" |
| 308 "BEGIN;\n" |
| 309 "CREATE TABLE kv(k INTEGER PRIMARY KEY, v BLOB);\n" |
| 310 "WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<%d)" |
| 311 " INSERT INTO kv(k,v) SELECT x, randomblob(%d+(random()%%(%d))) FROM c;\n" |
| 312 "COMMIT;\n", |
| 313 pgsz, nCount, sz, iVariance+1 |
| 314 ); |
| 315 rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg); |
| 316 if( rc ) fatalError("database create failed: %s", zErrMsg); |
| 317 sqlite3_free(zSql); |
| 318 sqlite3_close(db); |
| 319 return 0; |
| 320 } |
| 321 |
| 322 /* |
| 323 ** Analyze an existing database file. Report its content. |
| 324 */ |
| 325 static int statMain(int argc, char **argv){ |
| 326 char *zDb; |
| 327 int i, rc; |
| 328 sqlite3 *db; |
| 329 char *zSql; |
| 330 sqlite3_stmt *pStmt; |
| 331 |
| 332 assert( strcmp(argv[1],"stat")==0 ); |
| 333 assert( argc>=3 ); |
| 334 zDb = argv[2]; |
| 335 for(i=3; i<argc; i++){ |
| 336 char *z = argv[i]; |
| 337 if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z); |
| 338 if( z[1]=='-' ) z++; |
| 339 fatalError("unknown option: \"%s\"", argv[i]); |
| 340 } |
| 341 rc = sqlite3_open(zDb, &db); |
| 342 if( rc ){ |
| 343 fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db)); |
| 344 } |
| 345 zSql = sqlite3_mprintf( |
| 346 "SELECT count(*), min(length(v)), max(length(v)), avg(length(v))" |
| 347 " FROM kv" |
| 348 ); |
| 349 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); |
| 350 if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db)); |
| 351 sqlite3_free(zSql); |
| 352 if( sqlite3_step(pStmt)==SQLITE_ROW ){ |
| 353 printf("Number of entries: %8d\n", sqlite3_column_int(pStmt, 0)); |
| 354 printf("Average value size: %8d\n", sqlite3_column_int(pStmt, 3)); |
| 355 printf("Minimum value size: %8d\n", sqlite3_column_int(pStmt, 1)); |
| 356 printf("Maximum value size: %8d\n", sqlite3_column_int(pStmt, 2)); |
| 357 }else{ |
| 358 printf("No rows\n"); |
| 359 } |
| 360 sqlite3_finalize(pStmt); |
| 361 zSql = sqlite3_mprintf("PRAGMA page_size"); |
| 362 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); |
| 363 if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db)); |
| 364 sqlite3_free(zSql); |
| 365 if( sqlite3_step(pStmt)==SQLITE_ROW ){ |
| 366 printf("Page-size: %8d\n", sqlite3_column_int(pStmt, 0)); |
| 367 } |
| 368 sqlite3_finalize(pStmt); |
| 369 zSql = sqlite3_mprintf("PRAGMA page_count"); |
| 370 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); |
| 371 if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db)); |
| 372 sqlite3_free(zSql); |
| 373 if( sqlite3_step(pStmt)==SQLITE_ROW ){ |
| 374 printf("Page-count: %8d\n", sqlite3_column_int(pStmt, 0)); |
| 375 } |
| 376 sqlite3_finalize(pStmt); |
| 377 sqlite3_close(db); |
| 378 return 0; |
| 379 } |
| 380 |
| 381 /* |
| 382 ** Implementation of the "writefile(X,Y)" SQL function. The argument Y |
| 383 ** is written into file X. The number of bytes written is returned. Or |
| 384 ** NULL is returned if something goes wrong, such as being unable to open |
| 385 ** file X for writing. |
| 386 */ |
| 387 static void writefileFunc( |
| 388 sqlite3_context *context, |
| 389 int argc, |
| 390 sqlite3_value **argv |
| 391 ){ |
| 392 FILE *out; |
| 393 const char *z; |
| 394 sqlite3_int64 rc; |
| 395 const char *zFile; |
| 396 |
| 397 zFile = (const char*)sqlite3_value_text(argv[0]); |
| 398 if( zFile==0 ) return; |
| 399 out = fopen(zFile, "wb"); |
| 400 if( out==0 ) return; |
| 401 z = (const char*)sqlite3_value_blob(argv[1]); |
| 402 if( z==0 ){ |
| 403 rc = 0; |
| 404 }else{ |
| 405 rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out); |
| 406 } |
| 407 fclose(out); |
| 408 printf("\r%s ", zFile); fflush(stdout); |
| 409 sqlite3_result_int64(context, rc); |
| 410 } |
| 411 |
| 412 /* |
| 413 ** Export the kv table to individual files in the filesystem |
| 414 */ |
| 415 static int exportMain(int argc, char **argv){ |
| 416 char *zDb; |
| 417 char *zDir; |
| 418 sqlite3 *db; |
| 419 char *zSql; |
| 420 int rc; |
| 421 char *zErrMsg = 0; |
| 422 |
| 423 assert( strcmp(argv[1],"export")==0 ); |
| 424 assert( argc>=3 ); |
| 425 zDb = argv[2]; |
| 426 if( argc!=4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY"); |
| 427 zDir = argv[3]; |
| 428 if( pathType(zDir)!=PATH_DIR ){ |
| 429 fatalError("object \"%s\" is not a directory", zDir); |
| 430 } |
| 431 rc = sqlite3_open(zDb, &db); |
| 432 if( rc ){ |
| 433 fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db)); |
| 434 } |
| 435 sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0, |
| 436 writefileFunc, 0, 0); |
| 437 zSql = sqlite3_mprintf( |
| 438 "SELECT writefile(printf('%s/%%06d',k),v) FROM kv;", |
| 439 zDir |
| 440 ); |
| 441 rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg); |
| 442 if( rc ) fatalError("database create failed: %s", zErrMsg); |
| 443 sqlite3_free(zSql); |
| 444 sqlite3_close(db); |
| 445 printf("\n"); |
| 446 return 0; |
| 447 } |
| 448 |
| 449 /* |
| 450 ** Read the content of file zName into memory obtained from sqlite3_malloc64() |
| 451 ** and return a pointer to the buffer. The caller is responsible for freeing |
| 452 ** the memory. |
| 453 ** |
| 454 ** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes |
| 455 ** read. |
| 456 ** |
| 457 ** For convenience, a nul-terminator byte is always appended to the data read |
| 458 ** from the file before the buffer is returned. This byte is not included in |
| 459 ** the final value of (*pnByte), if applicable. |
| 460 ** |
| 461 ** NULL is returned if any error is encountered. The final value of *pnByte |
| 462 ** is undefined in this case. |
| 463 */ |
| 464 static unsigned char *readFile(const char *zName, int *pnByte){ |
| 465 FILE *in; /* FILE from which to read content of zName */ |
| 466 sqlite3_int64 nIn; /* Size of zName in bytes */ |
| 467 size_t nRead; /* Number of bytes actually read */ |
| 468 unsigned char *pBuf; /* Content read from disk */ |
| 469 |
| 470 nIn = fileSize(zName); |
| 471 if( nIn<0 ) return 0; |
| 472 in = fopen(zName, "rb"); |
| 473 if( in==0 ) return 0; |
| 474 pBuf = sqlite3_malloc64( nIn ); |
| 475 if( pBuf==0 ) return 0; |
| 476 nRead = fread(pBuf, (size_t)nIn, 1, in); |
| 477 fclose(in); |
| 478 if( nRead!=1 ){ |
| 479 sqlite3_free(pBuf); |
| 480 return 0; |
| 481 } |
| 482 if( pnByte ) *pnByte = (int)nIn; |
| 483 return pBuf; |
| 484 } |
| 485 |
| 486 /* |
| 487 ** Return the current time in milliseconds since the beginning of |
| 488 ** the Julian epoch. |
| 489 */ |
| 490 static sqlite3_int64 timeOfDay(void){ |
| 491 static sqlite3_vfs *clockVfs = 0; |
| 492 sqlite3_int64 t; |
| 493 if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); |
| 494 if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ |
| 495 clockVfs->xCurrentTimeInt64(clockVfs, &t); |
| 496 }else{ |
| 497 double r; |
| 498 clockVfs->xCurrentTime(clockVfs, &r); |
| 499 t = (sqlite3_int64)(r*86400000.0); |
| 500 } |
| 501 return t; |
| 502 } |
| 503 |
| 504 #ifdef __linux__ |
| 505 /* |
| 506 ** Attempt to display I/O stats on Linux using /proc/PID/io |
| 507 */ |
| 508 static void displayLinuxIoStats(FILE *out){ |
| 509 FILE *in; |
| 510 char z[200]; |
| 511 sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid()); |
| 512 in = fopen(z, "rb"); |
| 513 if( in==0 ) return; |
| 514 while( fgets(z, sizeof(z), in)!=0 ){ |
| 515 static const struct { |
| 516 const char *zPattern; |
| 517 const char *zDesc; |
| 518 } aTrans[] = { |
| 519 { "rchar: ", "Bytes received by read():" }, |
| 520 { "wchar: ", "Bytes sent to write():" }, |
| 521 { "syscr: ", "Read() system calls:" }, |
| 522 { "syscw: ", "Write() system calls:" }, |
| 523 { "read_bytes: ", "Bytes read from storage:" }, |
| 524 { "write_bytes: ", "Bytes written to storage:" }, |
| 525 { "cancelled_write_bytes: ", "Cancelled write bytes:" }, |
| 526 }; |
| 527 int i; |
| 528 for(i=0; i<sizeof(aTrans)/sizeof(aTrans[0]); i++){ |
| 529 int n = (int)strlen(aTrans[i].zPattern); |
| 530 if( strncmp(aTrans[i].zPattern, z, n)==0 ){ |
| 531 fprintf(out, "%-36s %s", aTrans[i].zDesc, &z[n]); |
| 532 break; |
| 533 } |
| 534 } |
| 535 } |
| 536 fclose(in); |
| 537 } |
| 538 #endif |
| 539 |
| 540 /* |
| 541 ** Display memory stats. |
| 542 */ |
| 543 static int display_stats( |
| 544 sqlite3 *db, /* Database to query */ |
| 545 int bReset /* True to reset SQLite stats */ |
| 546 ){ |
| 547 int iCur; |
| 548 int iHiwtr; |
| 549 FILE *out = stdout; |
| 550 |
| 551 fprintf(out, "\n"); |
| 552 |
| 553 iHiwtr = iCur = -1; |
| 554 sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset); |
| 555 fprintf(out, |
| 556 "Memory Used: %d (max %d) bytes\n", |
| 557 iCur, iHiwtr); |
| 558 iHiwtr = iCur = -1; |
| 559 sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset); |
| 560 fprintf(out, "Number of Outstanding Allocations: %d (max %d)\n", |
| 561 iCur, iHiwtr); |
| 562 iHiwtr = iCur = -1; |
| 563 sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset); |
| 564 fprintf(out, |
| 565 "Number of Pcache Pages Used: %d (max %d) pages\n", |
| 566 iCur, iHiwtr); |
| 567 iHiwtr = iCur = -1; |
| 568 sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset); |
| 569 fprintf(out, |
| 570 "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", |
| 571 iCur, iHiwtr); |
| 572 iHiwtr = iCur = -1; |
| 573 sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); |
| 574 fprintf(out, |
| 575 "Number of Scratch Allocations Used: %d (max %d)\n", |
| 576 iCur, iHiwtr); |
| 577 iHiwtr = iCur = -1; |
| 578 sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset); |
| 579 fprintf(out, |
| 580 "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", |
| 581 iCur, iHiwtr); |
| 582 iHiwtr = iCur = -1; |
| 583 sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset); |
| 584 fprintf(out, "Largest Allocation: %d bytes\n", |
| 585 iHiwtr); |
| 586 iHiwtr = iCur = -1; |
| 587 sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset); |
| 588 fprintf(out, "Largest Pcache Allocation: %d bytes\n", |
| 589 iHiwtr); |
| 590 iHiwtr = iCur = -1; |
| 591 sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, bReset); |
| 592 fprintf(out, "Largest Scratch Allocation: %d bytes\n", |
| 593 iHiwtr); |
| 594 |
| 595 iHiwtr = iCur = -1; |
| 596 sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); |
| 597 fprintf(out, "Pager Heap Usage: %d bytes\n", |
| 598 iCur); |
| 599 iHiwtr = iCur = -1; |
| 600 sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); |
| 601 fprintf(out, "Page cache hits: %d\n", iCur); |
| 602 iHiwtr = iCur = -1; |
| 603 sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); |
| 604 fprintf(out, "Page cache misses: %d\n", iCur); |
| 605 iHiwtr = iCur = -1; |
| 606 sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); |
| 607 fprintf(out, "Page cache writes: %d\n", iCur); |
| 608 iHiwtr = iCur = -1; |
| 609 |
| 610 #ifdef __linux__ |
| 611 displayLinuxIoStats(out); |
| 612 #endif |
| 613 |
| 614 return 0; |
| 615 } |
| 616 |
| 617 /* Blob access order */ |
| 618 #define ORDER_ASC 1 |
| 619 #define ORDER_DESC 2 |
| 620 #define ORDER_RANDOM 3 |
| 621 |
| 622 |
| 623 /* |
| 624 ** Run a performance test |
| 625 */ |
| 626 static int runMain(int argc, char **argv){ |
| 627 int eType; /* Is zDb a database or a directory? */ |
| 628 char *zDb; /* Database or directory name */ |
| 629 int i; /* Loop counter */ |
| 630 int rc; /* Return code from SQLite calls */ |
| 631 int nCount = 1000; /* Number of blob fetch operations */ |
| 632 int nExtra = 0; /* Extra cycles */ |
| 633 int iKey = 1; /* Next blob key */ |
| 634 int iMax = 0; /* Largest allowed key */ |
| 635 int iPagesize = 0; /* Database page size */ |
| 636 int iCache = 1000; /* Database cache size in kibibytes */ |
| 637 int bBlobApi = 0; /* Use the incremental blob I/O API */ |
| 638 int bStats = 0; /* Print stats before exiting */ |
| 639 int eOrder = ORDER_ASC; /* Access order */ |
| 640 sqlite3 *db = 0; /* Database connection */ |
| 641 sqlite3_stmt *pStmt = 0; /* Prepared statement for SQL access */ |
| 642 sqlite3_blob *pBlob = 0; /* Handle for incremental Blob I/O */ |
| 643 sqlite3_int64 tmStart; /* Start time */ |
| 644 sqlite3_int64 tmElapsed; /* Elapsed time */ |
| 645 int mmapSize = 0; /* --mmap N argument */ |
| 646 int nData = 0; /* Bytes of data */ |
| 647 sqlite3_int64 nTotal = 0; /* Total data read */ |
| 648 unsigned char *pData = 0; /* Content of the blob */ |
| 649 int nAlloc = 0; /* Space allocated for pData[] */ |
| 650 const char *zJMode = 0; /* Journal mode */ |
| 651 |
| 652 |
| 653 assert( strcmp(argv[1],"run")==0 ); |
| 654 assert( argc>=3 ); |
| 655 zDb = argv[2]; |
| 656 eType = pathType(zDb); |
| 657 if( eType==PATH_OTHER ) fatalError("unknown object type: \"%s\"", zDb); |
| 658 if( eType==PATH_NEXIST ) fatalError("object does not exist: \"%s\"", zDb); |
| 659 for(i=3; i<argc; i++){ |
| 660 char *z = argv[i]; |
| 661 if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z); |
| 662 if( z[1]=='-' ) z++; |
| 663 if( strcmp(z, "-count")==0 ){ |
| 664 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); |
| 665 nCount = integerValue(argv[++i]); |
| 666 if( nCount<1 ) fatalError("the --count must be positive"); |
| 667 continue; |
| 668 } |
| 669 if( strcmp(z, "-mmap")==0 ){ |
| 670 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); |
| 671 mmapSize = integerValue(argv[++i]); |
| 672 if( nCount<0 ) fatalError("the --mmap must be non-negative"); |
| 673 continue; |
| 674 } |
| 675 if( strcmp(z, "-max-id")==0 ){ |
| 676 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); |
| 677 iMax = integerValue(argv[++i]); |
| 678 continue; |
| 679 } |
| 680 if( strcmp(z, "-start")==0 ){ |
| 681 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); |
| 682 iKey = integerValue(argv[++i]); |
| 683 if( iKey<1 ) fatalError("the --start must be positive"); |
| 684 continue; |
| 685 } |
| 686 if( strcmp(z, "-cache-size")==0 ){ |
| 687 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); |
| 688 iCache = integerValue(argv[++i]); |
| 689 continue; |
| 690 } |
| 691 if( strcmp(z, "-jmode")==0 ){ |
| 692 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); |
| 693 zJMode = argv[++i]; |
| 694 continue; |
| 695 } |
| 696 if( strcmp(z, "-random")==0 ){ |
| 697 eOrder = ORDER_RANDOM; |
| 698 continue; |
| 699 } |
| 700 if( strcmp(z, "-asc")==0 ){ |
| 701 eOrder = ORDER_ASC; |
| 702 continue; |
| 703 } |
| 704 if( strcmp(z, "-desc")==0 ){ |
| 705 eOrder = ORDER_DESC; |
| 706 continue; |
| 707 } |
| 708 if( strcmp(z, "-blob-api")==0 ){ |
| 709 bBlobApi = 1; |
| 710 continue; |
| 711 } |
| 712 if( strcmp(z, "-stats")==0 ){ |
| 713 bStats = 1; |
| 714 continue; |
| 715 } |
| 716 fatalError("unknown option: \"%s\"", argv[i]); |
| 717 } |
| 718 tmStart = timeOfDay(); |
| 719 if( eType==PATH_DB ){ |
| 720 char *zSql; |
| 721 rc = sqlite3_open(zDb, &db); |
| 722 if( rc ){ |
| 723 fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db)); |
| 724 } |
| 725 zSql = sqlite3_mprintf("PRAGMA mmap_size=%d", mmapSize); |
| 726 sqlite3_exec(db, zSql, 0, 0, 0); |
| 727 zSql = sqlite3_mprintf("PRAGMA cache_size=%d", iCache); |
| 728 sqlite3_exec(db, zSql, 0, 0, 0); |
| 729 sqlite3_free(zSql); |
| 730 pStmt = 0; |
| 731 sqlite3_prepare_v2(db, "PRAGMA page_size", -1, &pStmt, 0); |
| 732 if( sqlite3_step(pStmt)==SQLITE_ROW ){ |
| 733 iPagesize = sqlite3_column_int(pStmt, 0); |
| 734 } |
| 735 sqlite3_finalize(pStmt); |
| 736 sqlite3_prepare_v2(db, "PRAGMA cache_size", -1, &pStmt, 0); |
| 737 if( sqlite3_step(pStmt)==SQLITE_ROW ){ |
| 738 iCache = sqlite3_column_int(pStmt, 0); |
| 739 }else{ |
| 740 iCache = 0; |
| 741 } |
| 742 sqlite3_finalize(pStmt); |
| 743 pStmt = 0; |
| 744 if( zJMode ){ |
| 745 zSql = sqlite3_mprintf("PRAGMA journal_mode=%Q", zJMode); |
| 746 sqlite3_exec(db, zSql, 0, 0, 0); |
| 747 sqlite3_free(zSql); |
| 748 } |
| 749 sqlite3_prepare_v2(db, "PRAGMA journal_mode", -1, &pStmt, 0); |
| 750 if( sqlite3_step(pStmt)==SQLITE_ROW ){ |
| 751 zJMode = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); |
| 752 }else{ |
| 753 zJMode = "???"; |
| 754 } |
| 755 sqlite3_finalize(pStmt); |
| 756 if( iMax<=0 ){ |
| 757 sqlite3_prepare_v2(db, "SELECT max(k) FROM kv", -1, &pStmt, 0); |
| 758 if( sqlite3_step(pStmt)==SQLITE_ROW ){ |
| 759 iMax = sqlite3_column_int(pStmt, 0); |
| 760 } |
| 761 sqlite3_finalize(pStmt); |
| 762 } |
| 763 pStmt = 0; |
| 764 sqlite3_exec(db, "BEGIN", 0, 0, 0); |
| 765 } |
| 766 if( iMax<=0 ) iMax = 1000; |
| 767 for(i=0; i<nCount; i++){ |
| 768 if( eType==PATH_DIR ){ |
| 769 /* CASE 1: Reading blobs out of separate files */ |
| 770 char *zKey; |
| 771 zKey = sqlite3_mprintf("%s/%06d", zDb, iKey); |
| 772 nData = 0; |
| 773 pData = readFile(zKey, &nData); |
| 774 sqlite3_free(zKey); |
| 775 sqlite3_free(pData); |
| 776 }else if( bBlobApi ){ |
| 777 /* CASE 2: Reading from database using the incremental BLOB I/O API */ |
| 778 if( pBlob==0 ){ |
| 779 rc = sqlite3_blob_open(db, "main", "kv", "v", iKey, 0, &pBlob); |
| 780 if( rc ){ |
| 781 fatalError("could not open sqlite3_blob handle: %s", |
| 782 sqlite3_errmsg(db)); |
| 783 } |
| 784 }else{ |
| 785 rc = sqlite3_blob_reopen(pBlob, iKey); |
| 786 } |
| 787 if( rc==SQLITE_OK ){ |
| 788 nData = sqlite3_blob_bytes(pBlob); |
| 789 if( nAlloc<nData+1 ){ |
| 790 nAlloc = nData+100; |
| 791 pData = sqlite3_realloc(pData, nAlloc); |
| 792 } |
| 793 if( pData==0 ) fatalError("cannot allocate %d bytes", nData+1); |
| 794 rc = sqlite3_blob_read(pBlob, pData, nData, 0); |
| 795 if( rc!=SQLITE_OK ){ |
| 796 fatalError("could not read the blob at %d: %s", iKey, |
| 797 sqlite3_errmsg(db)); |
| 798 } |
| 799 } |
| 800 }else{ |
| 801 /* CASE 3: Reading from database using SQL */ |
| 802 if( pStmt==0 ){ |
| 803 rc = sqlite3_prepare_v2(db, |
| 804 "SELECT v FROM kv WHERE k=?1", -1, &pStmt, 0); |
| 805 if( rc ){ |
| 806 fatalError("cannot prepare query: %s", sqlite3_errmsg(db)); |
| 807 } |
| 808 }else{ |
| 809 sqlite3_reset(pStmt); |
| 810 } |
| 811 sqlite3_bind_int(pStmt, 1, iKey); |
| 812 rc = sqlite3_step(pStmt); |
| 813 if( rc==SQLITE_ROW ){ |
| 814 nData = sqlite3_column_bytes(pStmt, 0); |
| 815 pData = (unsigned char*)sqlite3_column_blob(pStmt, 0); |
| 816 }else{ |
| 817 nData = 0; |
| 818 } |
| 819 } |
| 820 if( eOrder==ORDER_ASC ){ |
| 821 iKey++; |
| 822 if( iKey>iMax ) iKey = 1; |
| 823 }else if( eOrder==ORDER_DESC ){ |
| 824 iKey--; |
| 825 if( iKey<=0 ) iKey = iMax; |
| 826 }else{ |
| 827 iKey = (randInt()%iMax)+1; |
| 828 } |
| 829 nTotal += nData; |
| 830 if( nData==0 ){ nCount++; nExtra++; } |
| 831 } |
| 832 if( nAlloc ) sqlite3_free(pData); |
| 833 if( pStmt ) sqlite3_finalize(pStmt); |
| 834 if( pBlob ) sqlite3_blob_close(pBlob); |
| 835 if( bStats ){ |
| 836 display_stats(db, 0); |
| 837 } |
| 838 if( db ) sqlite3_close(db); |
| 839 tmElapsed = timeOfDay() - tmStart; |
| 840 if( nExtra ){ |
| 841 printf("%d cycles due to %d misses\n", nCount, nExtra); |
| 842 } |
| 843 if( eType==PATH_DB ){ |
| 844 printf("SQLite version: %s\n", sqlite3_libversion()); |
| 845 } |
| 846 printf("--count %d --max-id %d", nCount-nExtra, iMax); |
| 847 switch( eOrder ){ |
| 848 case ORDER_RANDOM: printf(" --random\n"); break; |
| 849 case ORDER_DESC: printf(" --desc\n"); break; |
| 850 default: printf(" --asc\n"); break; |
| 851 } |
| 852 if( eType==PATH_DB ){ |
| 853 printf("--cache-size %d --jmode %s\n", iCache, zJMode); |
| 854 printf("--mmap %d%s\n", mmapSize, bBlobApi ? " --blob-api" : ""); |
| 855 } |
| 856 if( iPagesize ) printf("Database page size: %d\n", iPagesize); |
| 857 printf("Total elapsed time: %.3f\n", tmElapsed/1000.0); |
| 858 printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount); |
| 859 printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed)); |
| 860 return 0; |
| 861 } |
| 862 |
| 863 |
| 864 int main(int argc, char **argv){ |
| 865 if( argc<3 ) showHelp(); |
| 866 if( strcmp(argv[1],"init")==0 ){ |
| 867 return initMain(argc, argv); |
| 868 } |
| 869 if( strcmp(argv[1],"export")==0 ){ |
| 870 return exportMain(argc, argv); |
| 871 } |
| 872 if( strcmp(argv[1],"run")==0 ){ |
| 873 return runMain(argc, argv); |
| 874 } |
| 875 if( strcmp(argv[1],"stat")==0 ){ |
| 876 return statMain(argc, argv); |
| 877 } |
| 878 showHelp(); |
| 879 return 0; |
| 880 } |
OLD | NEW |