@ -18,6 +18,7 @@ int main() {
# include <vector>
# include <gflags/gflags.h>
# include "db/db_impl.h"
# include "rocksdb/comparator.h"
# include "rocksdb/db.h"
# include "rocksdb/filter_policy.h"
@ -26,9 +27,11 @@ int main() {
# include "rocksdb/slice_transform.h"
# include "rocksdb/table.h"
# include "util/histogram.h"
# include "util/random.h"
# include "util/stop_watch.h"
# include "util/string_util.h"
# include "util/testharness.h"
# include "utilities/merge_operators.h"
using GFLAGS : : ParseCommandLineFlags ;
@ -46,6 +49,7 @@ DEFINE_int32(skiplist_height, 4, "");
DEFINE_double ( memtable_prefix_bloom_size_ratio , 0.1 , " " ) ;
DEFINE_int32 ( memtable_huge_page_size , 2 * 1024 * 1024 , " " ) ;
DEFINE_int32 ( value_size , 40 , " " ) ;
DEFINE_bool ( enable_print , false , " Print options generated to console. " ) ;
// Path to the database on file system
const std : : string kDbName = rocksdb : : test : : TmpDir ( ) + " /prefix_test " ;
@ -106,6 +110,10 @@ class TestKeyComparator : public Comparator {
return 0 ;
}
bool operator ( ) ( const TestKey & a , const TestKey & b ) const {
return Compare ( TestKeyToSlice ( a ) , TestKeyToSlice ( b ) ) < 0 ;
}
virtual const char * Name ( ) const override {
return " TestKeyComparator " ;
}
@ -124,6 +132,23 @@ void PutKey(DB* db, WriteOptions write_options, uint64_t prefix,
ASSERT_OK ( db - > Put ( write_options , key , value ) ) ;
}
void PutKey ( DB * db , WriteOptions write_options , const TestKey & test_key ,
const Slice & value ) {
Slice key = TestKeyToSlice ( test_key ) ;
ASSERT_OK ( db - > Put ( write_options , key , value ) ) ;
}
void MergeKey ( DB * db , WriteOptions write_options , const TestKey & test_key ,
const Slice & value ) {
Slice key = TestKeyToSlice ( test_key ) ;
ASSERT_OK ( db - > Merge ( write_options , key , value ) ) ;
}
void DeleteKey ( DB * db , WriteOptions write_options , const TestKey & test_key ) {
Slice key = TestKeyToSlice ( test_key ) ;
ASSERT_OK ( db - > Delete ( write_options , key ) ) ;
}
void SeekIterator ( Iterator * iter , uint64_t prefix , uint64_t suffix ) {
TestKey test_key ( prefix , suffix ) ;
Slice key = TestKeyToSlice ( test_key ) ;
@ -629,8 +654,206 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
}
}
TEST_F ( PrefixTest , PrefixSeekModePrev ) {
// Only for SkipListFactory
options . memtable_factory . reset ( new SkipListFactory ) ;
options . merge_operator = MergeOperators : : CreatePutOperator ( ) ;
options . write_buffer_size = 1024 * 1024 ;
Random rnd ( 1 ) ;
for ( size_t m = 1 ; m < 100 ; m + + ) {
std : : cout < < " [ " + std : : to_string ( m ) + " ] " + " *** Mem table: "
< < options . memtable_factory - > Name ( ) < < std : : endl ;
DestroyDB ( kDbName , Options ( ) ) ;
auto db = OpenDb ( ) ;
WriteOptions write_options ;
ReadOptions read_options ;
std : : map < TestKey , std : : string , TestKeyComparator > entry_maps [ 3 ] , whole_map ;
for ( uint64_t i = 0 ; i < 10 ; i + + ) {
int div = i % 3 + 1 ;
for ( uint64_t j = 0 ; j < 10 ; j + + ) {
whole_map [ TestKey ( i , j ) ] = entry_maps [ rnd . Uniform ( div ) ] [ TestKey ( i , j ) ] =
' v ' + std : : to_string ( i ) + std : : to_string ( j ) ;
}
}
std : : map < TestKey , std : : string , TestKeyComparator > type_map ;
for ( size_t i = 0 ; i < 3 ; i + + ) {
for ( auto & kv : entry_maps [ i ] ) {
if ( rnd . OneIn ( 3 ) ) {
PutKey ( db . get ( ) , write_options , kv . first , kv . second ) ;
type_map [ kv . first ] = " value " ;
} else {
MergeKey ( db . get ( ) , write_options , kv . first , kv . second ) ;
type_map [ kv . first ] = " merge " ;
}
}
if ( i < 2 ) {
db - > Flush ( FlushOptions ( ) ) ;
}
}
for ( size_t i = 0 ; i < 2 ; i + + ) {
for ( auto & kv : entry_maps [ i ] ) {
if ( rnd . OneIn ( 10 ) ) {
whole_map . erase ( kv . first ) ;
DeleteKey ( db . get ( ) , write_options , kv . first ) ;
entry_maps [ 2 ] [ kv . first ] = " delete " ;
}
}
}
if ( FLAGS_enable_print ) {
for ( size_t i = 0 ; i < 3 ; i + + ) {
for ( auto & kv : entry_maps [ i ] ) {
std : : cout < < " [ " < < i < < " ] " < < kv . first . prefix < < kv . first . sorted
< < " " < < kv . second + " " + type_map [ kv . first ] < < std : : endl ;
}
}
}
std : : unique_ptr < Iterator > iter ( db - > NewIterator ( read_options ) ) ;
for ( uint64_t prefix = 0 ; prefix < 10 ; prefix + + ) {
uint64_t start_suffix = rnd . Uniform ( 9 ) ;
SeekIterator ( iter . get ( ) , prefix , start_suffix ) ;
auto it = whole_map . find ( TestKey ( prefix , start_suffix ) ) ;
if ( it = = whole_map . end ( ) ) {
continue ;
}
ASSERT_NE ( it , whole_map . end ( ) ) ;
ASSERT_TRUE ( iter - > Valid ( ) ) ;
if ( FLAGS_enable_print ) {
std : : cout < < " round " < < prefix
< < " iter: " < < SliceToTestKey ( iter - > key ( ) ) - > prefix
< < SliceToTestKey ( iter - > key ( ) ) - > sorted
< < " | map: " < < it - > first . prefix < < it - > first . sorted < < " | "
< < iter - > value ( ) . ToString ( ) < < " " < < it - > second < < std : : endl ;
}
ASSERT_EQ ( iter - > value ( ) , it - > second ) ;
uint64_t stored_prefix = prefix ;
for ( size_t k = 0 ; k < 9 ; k + + ) {
if ( rnd . OneIn ( 2 ) | | it = = whole_map . begin ( ) ) {
iter - > Next ( ) ;
it + + ;
if ( FLAGS_enable_print ) {
std : : cout < < " Next >> " ;
}
} else {
iter - > Prev ( ) ;
it - - ;
if ( FLAGS_enable_print ) {
std : : cout < < " Prev >> " ;
}
}
if ( ! iter - > Valid ( ) | |
SliceToTestKey ( iter - > key ( ) ) - > prefix ! = stored_prefix ) {
break ;
}
stored_prefix = SliceToTestKey ( iter - > key ( ) ) - > prefix ;
ASSERT_TRUE ( iter - > Valid ( ) ) ;
ASSERT_NE ( it , whole_map . end ( ) ) ;
ASSERT_EQ ( iter - > value ( ) , it - > second ) ;
if ( FLAGS_enable_print ) {
std : : cout < < " iter: " < < SliceToTestKey ( iter - > key ( ) ) - > prefix
< < SliceToTestKey ( iter - > key ( ) ) - > sorted
< < " | map: " < < it - > first . prefix < < it - > first . sorted
< < " | " < < iter - > value ( ) . ToString ( ) < < " " < < it - > second
< < std : : endl ;
}
}
}
}
}
TEST_F ( PrefixTest , PrefixSeekModePrev2 ) {
// Only for SkipListFactory
// test the case
// iter1 iter2
// | prefix | suffix | | prefix | suffix |
// | 1 | 1 | | 1 | 2 |
// | 1 | 3 | | 1 | 4 |
// | 2 | 1 | | 3 | 3 |
// | 2 | 2 | | 3 | 4 |
// after seek(15), iter1 will be at 21 and iter2 will be 33.
// Then if call Prev() in prefix mode where SeekForPrev(21) gets called,
// iter2 should turn to invalid state because of bloom filter.
options . memtable_factory . reset ( new SkipListFactory ) ;
options . write_buffer_size = 1024 * 1024 ;
std : : string v13 ( " v13 " ) ;
DestroyDB ( kDbName , Options ( ) ) ;
auto db = OpenDb ( ) ;
WriteOptions write_options ;
ReadOptions read_options ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 2 ) , " v12 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 4 ) , " v14 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 3 , 3 ) , " v33 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 3 , 4 ) , " v34 " ) ;
db - > Flush ( FlushOptions ( ) ) ;
reinterpret_cast < DBImpl * > ( db . get ( ) ) - > TEST_WaitForFlushMemTable ( ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 1 ) , " v11 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 3 ) , " v13 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 2 , 1 ) , " v21 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 2 , 2 ) , " v22 " ) ;
db - > Flush ( FlushOptions ( ) ) ;
reinterpret_cast < DBImpl * > ( db . get ( ) ) - > TEST_WaitForFlushMemTable ( ) ;
std : : unique_ptr < Iterator > iter ( db - > NewIterator ( read_options ) ) ;
SeekIterator ( iter . get ( ) , 1 , 5 ) ;
iter - > Prev ( ) ;
ASSERT_EQ ( iter - > value ( ) , v13 ) ;
}
TEST_F ( PrefixTest , PrefixSeekModePrev3 ) {
// Only for SkipListFactory
// test SeekToLast() with iterate_upper_bound_ in prefix_seek_mode
options . memtable_factory . reset ( new SkipListFactory ) ;
options . write_buffer_size = 1024 * 1024 ;
std : : string v14 ( " v14 " ) ;
TestKey upper_bound_key = TestKey ( 1 , 5 ) ;
Slice upper_bound = TestKeyToSlice ( upper_bound_key ) ;
{
DestroyDB ( kDbName , Options ( ) ) ;
auto db = OpenDb ( ) ;
WriteOptions write_options ;
ReadOptions read_options ;
read_options . iterate_upper_bound = & upper_bound ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 2 ) , " v12 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 4 ) , " v14 " ) ;
db - > Flush ( FlushOptions ( ) ) ;
reinterpret_cast < DBImpl * > ( db . get ( ) ) - > TEST_WaitForFlushMemTable ( ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 1 ) , " v11 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 3 ) , " v13 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 2 , 1 ) , " v21 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 2 , 2 ) , " v22 " ) ;
db - > Flush ( FlushOptions ( ) ) ;
reinterpret_cast < DBImpl * > ( db . get ( ) ) - > TEST_WaitForFlushMemTable ( ) ;
std : : unique_ptr < Iterator > iter ( db - > NewIterator ( read_options ) ) ;
iter - > SeekToLast ( ) ;
ASSERT_EQ ( iter - > value ( ) , v14 ) ;
}
{
DestroyDB ( kDbName , Options ( ) ) ;
auto db = OpenDb ( ) ;
WriteOptions write_options ;
ReadOptions read_options ;
read_options . iterate_upper_bound = & upper_bound ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 2 ) , " v12 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 4 ) , " v14 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 3 , 3 ) , " v33 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 3 , 4 ) , " v34 " ) ;
db - > Flush ( FlushOptions ( ) ) ;
reinterpret_cast < DBImpl * > ( db . get ( ) ) - > TEST_WaitForFlushMemTable ( ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 1 ) , " v11 " ) ;
PutKey ( db . get ( ) , write_options , TestKey ( 1 , 3 ) , " v13 " ) ;
db - > Flush ( FlushOptions ( ) ) ;
reinterpret_cast < DBImpl * > ( db . get ( ) ) - > TEST_WaitForFlushMemTable ( ) ;
std : : unique_ptr < Iterator > iter ( db - > NewIterator ( read_options ) ) ;
iter - > SeekToLast ( ) ;
ASSERT_EQ ( iter - > value ( ) , v14 ) ;
}
}
} // end namespace rocksdb
int main ( int argc , char * * argv ) {
: : testing : : InitGoogleTest ( & argc , argv ) ;
ParseCommandLineFlags ( & argc , & argv , true ) ;
Aaron Gao
8 years ago