000001  # 2014 May 6.
000002  #
000003  # The author disclaims copyright to this source code.  In place of
000004  # a legal notice, here is a blessing:
000005  #
000006  #    May you do good and not evil.
000007  #    May you find forgiveness for yourself and forgive others.
000008  #    May you share freely, never taking more than you give.
000009  #
000010  #***********************************************************************
000011  # This file implements regression tests for SQLite library. 
000012  #
000013  # The tests in this file are brute force tests of the multi-threaded
000014  # sorter.
000015  #
000016  
000017  set testdir [file dirname $argv0]
000018  source $testdir/tester.tcl
000019  set testprefix sort4
000020  db close
000021  sqlite3_shutdown
000022  sqlite3_config_pmasz 10
000023  sqlite3_initialize
000024  sqlite3 db test.db
000025  
000026  
000027  # Configure the sorter to use 3 background threads.
000028  #
000029  # EVIDENCE-OF: R-19249-32353 SQLITE_LIMIT_WORKER_THREADS The maximum
000030  # number of auxiliary worker threads that a single prepared statement
000031  # may start.
000032  #
000033  do_test sort4-init001 {
000034    db eval {PRAGMA threads=5}
000035    sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS -1
000036  } {5}
000037  do_test sort4-init002 {
000038    sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS 3
000039    db eval {PRAGMA threads}
000040  } {3}
000041  
000042  
000043  # Minimum number of seconds to run for. If the value is 0, each test
000044  # is run exactly once. Otherwise, tests are repeated until the timeout
000045  # expires.
000046  set SORT4TIMEOUT 0
000047  if {[permutation] == "multithread"} { set SORT4TIMEOUT 300 }
000048  
000049  #--------------------------------------------------------------------
000050  # Set up a table "t1" containing $nRow rows. Each row contains also
000051  # contains blob fields that collectively contain at least $nPayload 
000052  # bytes of content. The table schema is as follows:
000053  #
000054  #   CREATE TABLE t1(a INTEGER, <extra-columns>, b INTEGER);
000055  #
000056  # For each row, the values of columns "a" and "b" are set to the same
000057  # pseudo-randomly selected integer. The "extra-columns", of which there
000058  # are at most eight, are named c0, c1, c2 etc. Column c0 contains a 4
000059  # byte string. Column c1 an 8 byte string. Field c2 16 bytes, and so on.
000060  #
000061  # This table is intended to be used for testing queries of the form: 
000062  #
000063  #   SELECT a, <cols>, b FROM t1 ORDER BY a;
000064  #
000065  # The test code checks that rows are returned in order, and that the 
000066  # values of "a" and "b" are the same for each row (the idea being that
000067  # if field "b" at the end of the sorter record has not been corrupted, 
000068  # the rest of the record is probably Ok as well).
000069  #
000070  proc populate_table {nRow nPayload} {
000071    set nCol 0
000072  
000073    set n 0
000074    for {set nCol 0} {$n < $nPayload} {incr nCol} {
000075      incr n [expr (4 << $nCol)]
000076    }
000077  
000078    set cols [lrange [list xxx c0 c1 c2 c3 c4 c5 c6 c7] 1 $nCol]
000079    set data [lrange [list xxx \
000080        randomblob(4) randomblob(8) randomblob(16) randomblob(32) \
000081        randomblob(64) randomblob(128) randomblob(256) randomblob(512) \
000082    ] 1 $nCol]
000083  
000084    execsql { DROP TABLE IF EXISTS t1 }
000085  
000086    db transaction {
000087      execsql "CREATE TABLE t1(a, [join $cols ,], b);"
000088      set insert "INSERT INTO t1 VALUES(:k, [join $data ,], :k)"
000089      for {set i 0} {$i < $nRow} {incr i} {
000090        set k [expr int(rand()*1000000000)]
000091        execsql $insert
000092      }
000093    }
000094  }
000095  
000096  # Helper for [do_sorter_test]
000097  #
000098  proc sorter_test {nRow nRead nPayload} {
000099    set res [list]
000100  
000101    set nLoad [expr ($nRow > $nRead) ? $nRead : $nRow]
000102  
000103    set nPayload [expr (($nPayload+3)/4) * 4]
000104    set cols [list]
000105    foreach {mask col} { 
000106      0x04  c0 0x08  c1 0x10  c2 0x20  c3 
000107      0x40  c4 0x80  c5 0x100 c6 0x200 c7 
000108    } {
000109      if {$nPayload & $mask} { lappend cols $col }
000110    }
000111  
000112    # Create two SELECT statements. Statement $sql1 uses the sorter to sort
000113    # $nRow records of a bit over $nPayload bytes each read from the "t1"
000114    # table created by [populate_table] proc above. Rows are sorted in order
000115    # of the integer field in each "t1" record.
000116    #
000117    # The second SQL statement sorts the same set of rows as the first, but
000118    # uses a LIMIT clause, causing SQLite to use a temp table instead of the
000119    # sorter for sorting.
000120    #
000121    set sql1 "SELECT a, [join $cols ,], b FROM t1 WHERE rowid<=$nRow ORDER BY a"
000122    set sql2 "SELECT a FROM t1 WHERE rowid<=$nRow ORDER BY a LIMIT $nRead"
000123  
000124    # Pass the two SQL statements to a helper command written in C. This
000125    # command steps statement $sql1 $nRead times and compares the integer
000126    # values in the rows returned with the results of executing $sql2. If
000127    # the comparison fails (indicating some bug in the sorter), a Tcl
000128    # exception is thrown.
000129    #
000130    sorter_test_sort4_helper db $sql1 $nRead $sql2
000131    set {} {} 
000132  }
000133  
000134  # Usage:
000135  #
000136  #   do_sorter_test <testname> <args>...
000137  #
000138  # where <args> are any of the following switches:
000139  #
000140  #   -rows N          (number of rows to have sorter sort)
000141  #   -read N          (number of rows to read out of sorter)
000142  #   -payload N       (bytes of payload to read with each row)
000143  #   -cachesize N     (Value for "PRAGMA cache_size = ?")
000144  #   -repeats N       (number of times to repeat test)
000145  #   -fakeheap BOOL   (true to use separate allocations for in-memory records)
000146  #
000147  proc do_sorter_test {tn args} {
000148    set a(-rows)      1000
000149    set a(-repeats)   1
000150    set a(-read)      100
000151    set a(-payload)   100
000152    set a(-cachesize) 100
000153    set a(-fakeheap)  0
000154  
000155    foreach {s val} $args {
000156      if {[info exists a($s)]==0} { 
000157        unset a(-cachesize)
000158        set optlist "[join [array names a] ,] or -cachesize"
000159        error "Unknown option $s, expected $optlist"
000160      }
000161      set a($s) $val
000162    }
000163    if {[permutation] == "memsys3" || [permutation] == "memsys5"} {
000164      set a(-fakeheap) 0
000165    }
000166    if {$a(-fakeheap)} { sorter_test_fakeheap 1 }
000167  
000168  
000169    db eval "PRAGMA cache_size = $a(-cachesize)"
000170    do_test $tn [subst -nocommands {
000171      for {set i 0} {[set i] < $a(-repeats)} {incr i} {
000172        sorter_test $a(-rows) $a(-read) $a(-payload)
000173      }
000174    }] {}
000175  
000176    if {$a(-fakeheap)} { sorter_test_fakeheap 0 }
000177  }
000178  
000179  proc clock_seconds {} {
000180    db one {SELECT strftime('%s')}
000181  }
000182  
000183  #-------------------------------------------------------------------------
000184  # Begin tests here.
000185  
000186  # Create a test database.
000187  do_test 1 {
000188    execsql "PRAGMA page_size = 4096"
000189    populate_table 100000 500
000190  } {}
000191  
000192  set iTimeLimit [expr [clock_seconds] + $SORT4TIMEOUT]
000193  
000194  for {set t 2} {1} {incr tn} {
000195    do_sorter_test $t.2 -repeats 10 -rows 1000   -read 100
000196    do_sorter_test $t.3 -repeats 10 -rows 100000 -read 1000
000197    do_sorter_test $t.4 -repeats 10 -rows 100000 -read 1000 -payload 500
000198    do_sorter_test $t.5 -repeats 10 -rows 100000 -read 100000 -payload 8
000199    do_sorter_test $t.6 -repeats 10 -rows 100000 -read 10 -payload 8
000200    do_sorter_test $t.7 -repeats 10 -rows 10000 -read 10000 -payload 8 -fakeheap 1
000201    do_sorter_test $t.8 -repeats 10 -rows 100000 -read 10000 -cachesize 250
000202  
000203    set iNow [clock_seconds]
000204    if {$iNow>=$iTimeLimit} break
000205    do_test "$testprefix-([expr $iTimeLimit-$iNow] seconds remain)" {} {}
000206  }
000207  
000208  finish_test