NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Extend some tests to format_version=3 (#3942)

Browse Source 
Summary:
format_version=3 changes the format of SST index. This is however not being tested currently since tests only work with the default format_version which is currently 2. The patch extends the most related tests to also test for format_version=3.
Closes https://github.com/facebook/rocksdb/pull/3942

Differential Revision: D8238413

Pulled By: maysamyabandeh

fbshipit-source-id: 915725f55753dd8e9188e802bf471c23645ad035
main
Maysam Yabandeh 7 years ago committed by Facebook Github Bot
parent 2210152947
commit d0c38c0c8c
12 changed files with 266 additions and 157 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 199
      db/column_family_test.cc
  2. 40
      db/comparator_db_test.cc
  3. 9
      db/db_test_util.cc
  4. 3
      db/db_test_util.h
  5. 15
      table/block_based_table_reader.cc
  6. 2
      table/format.h
  7. 119
      table/table_test.cc
  8. 8
      tools/db_bench_tool.cc
  9. 1
      tools/db_crashtest.py
  10. 12
      tools/db_stress.cc
  11. 12
      util/testutil.cc
  12. 3
      util/testutil.h

199
db/column_family_test.cc
Unescape View File

@ -57,9 +57,9 @@ class EnvCounter : public EnvWrapper {
std::atomic<int> num_new_writable_file_; std::atomic<int> num_new_writable_file_;
}; };
class ColumnFamilyTest : public testing::Test { class ColumnFamilyTestBase : public testing::Test {
public: public:
ColumnFamilyTest() : rnd_(139) { ColumnFamilyTestBase(uint32_t format) : rnd_(139), format_(format) {
env_ = new EnvCounter(Env::Default()); env_ = new EnvCounter(Env::Default());
dbname_ = test::TmpDir() + "/column_family_test"; dbname_ = test::TmpDir() + "/column_family_test";
db_options_.create_if_missing = true; db_options_.create_if_missing = true;
@ -68,7 +68,7 @@ class ColumnFamilyTest : public testing::Test {
DestroyDB(dbname_, Options(db_options_, column_family_options_)); DestroyDB(dbname_, Options(db_options_, column_family_options_));
} }
~ColumnFamilyTest() { virtual ~ColumnFamilyTestBase() {
std::vector<ColumnFamilyDescriptor> column_families; std::vector<ColumnFamilyDescriptor> column_families;
for (auto h : handles_) { for (auto h : handles_) {
ColumnFamilyDescriptor cfdescriptor; ColumnFamilyDescriptor cfdescriptor;
@ -81,6 +81,12 @@ class ColumnFamilyTest : public testing::Test {
delete env_; delete env_;
} }
BlockBasedTableOptions GetBlockBasedTableOptions() {
BlockBasedTableOptions options;
options.format_version = format_;
return options;
}
// Return the value to associate with the specified key // Return the value to associate with the specified key
Slice Value(int k, std::string* storage) { Slice Value(int k, std::string* storage) {
if (k == 0) { if (k == 0) {
@ -516,9 +522,22 @@ class ColumnFamilyTest : public testing::Test {
DB* db_ = nullptr; DB* db_ = nullptr;
EnvCounter* env_; EnvCounter* env_;
Random rnd_; Random rnd_;
uint32_t format_;
};
class ColumnFamilyTest
: public ColumnFamilyTestBase,
virtual public ::testing::WithParamInterface<uint32_t> {
public:
ColumnFamilyTest() : ColumnFamilyTestBase(GetParam()) {}
}; };
TEST_F(ColumnFamilyTest, DontReuseColumnFamilyID) { INSTANTIATE_TEST_CASE_P(FormatDef, ColumnFamilyTest,
testing::Values(test::kDefaultFormatVersion));
INSTANTIATE_TEST_CASE_P(FormatLatest, ColumnFamilyTest,
testing::Values(test::kLatestFormatVersion));
TEST_P(ColumnFamilyTest, DontReuseColumnFamilyID) {
for (int iter = 0; iter < 3; ++iter) { for (int iter = 0; iter < 3; ++iter) {
Open(); Open();
CreateColumnFamilies({"one", "two", "three"}); CreateColumnFamilies({"one", "two", "three"});
@ -537,7 +556,8 @@ TEST_F(ColumnFamilyTest, DontReuseColumnFamilyID) {
Reopen(); Reopen();
} }
CreateColumnFamilies({"three2"}); CreateColumnFamilies({"three2"});
// ID 3 that was used for dropped column family "three" should not be reused // ID 3 that was used for dropped column family "three" should not be
// reused
auto cfh3 = reinterpret_cast<ColumnFamilyHandleImpl*>(handles_[3]); auto cfh3 = reinterpret_cast<ColumnFamilyHandleImpl*>(handles_[3]);
ASSERT_EQ(4U, cfh3->GetID()); ASSERT_EQ(4U, cfh3->GetID());
Close(); Close();
@ -546,7 +566,7 @@ TEST_F(ColumnFamilyTest, DontReuseColumnFamilyID) {
} }
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
TEST_F(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) { TEST_P(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) {
Open(); Open();
rocksdb::SyncPoint::GetInstance()->LoadDependency( rocksdb::SyncPoint::GetInstance()->LoadDependency(
@ -569,10 +589,13 @@ TEST_F(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) {
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
class FlushEmptyCFTestWithParam : public ColumnFamilyTest, class FlushEmptyCFTestWithParam
public testing::WithParamInterface<bool> { : public ColumnFamilyTestBase,
virtual public testing::WithParamInterface<std::tuple<uint32_t, bool>> {
public: public:
FlushEmptyCFTestWithParam() { allow_2pc_ = GetParam(); } FlushEmptyCFTestWithParam()
: ColumnFamilyTestBase(std::get<0>(GetParam())),
allow_2pc_(std::get<1>(GetParam())) {}
// Required if inheriting from testing::WithParamInterface<> // Required if inheriting from testing::WithParamInterface<>
static void SetUpTestCase() {} static void SetUpTestCase() {}
@ -697,10 +720,16 @@ TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest2) {
db_options_.env = env_; db_options_.env = env_;
} }
INSTANTIATE_TEST_CASE_P(FlushEmptyCFTestWithParam, FlushEmptyCFTestWithParam, INSTANTIATE_TEST_CASE_P(
::testing::Bool()); FormatDef, FlushEmptyCFTestWithParam,
testing::Values(std::make_tuple(test::kDefaultFormatVersion, true),
std::make_tuple(test::kDefaultFormatVersion, false)));
INSTANTIATE_TEST_CASE_P(
FormatLatest, FlushEmptyCFTestWithParam,
testing::Values(std::make_tuple(test::kLatestFormatVersion, true),
std::make_tuple(test::kLatestFormatVersion, false)));
TEST_F(ColumnFamilyTest, AddDrop) { TEST_P(ColumnFamilyTest, AddDrop) {
Open(); Open();
CreateColumnFamilies({"one", "two", "three"}); CreateColumnFamilies({"one", "two", "three"});
ASSERT_EQ("NOT_FOUND", Get(1, "fodor")); ASSERT_EQ("NOT_FOUND", Get(1, "fodor"));
@ -726,7 +755,7 @@ TEST_F(ColumnFamilyTest, AddDrop) {
std::vector<std::string>({"default", "four", "three"})); std::vector<std::string>({"default", "four", "three"}));
} }
TEST_F(ColumnFamilyTest, BulkAddDrop) { TEST_P(ColumnFamilyTest, BulkAddDrop) {
constexpr int kNumCF = 1000; constexpr int kNumCF = 1000;
ColumnFamilyOptions cf_options; ColumnFamilyOptions cf_options;
WriteOptions write_options; WriteOptions write_options;
@ -764,7 +793,7 @@ TEST_F(ColumnFamilyTest, BulkAddDrop) {
ASSERT_TRUE(families == std::vector<std::string>({"default"})); ASSERT_TRUE(families == std::vector<std::string>({"default"}));
} }
TEST_F(ColumnFamilyTest, DropTest) { TEST_P(ColumnFamilyTest, DropTest) {
// first iteration - dont reopen DB before dropping // first iteration - dont reopen DB before dropping
// second iteration - reopen DB before dropping // second iteration - reopen DB before dropping
for (int iter = 0; iter < 2; ++iter) { for (int iter = 0; iter < 2; ++iter) {
@ -788,7 +817,7 @@ TEST_F(ColumnFamilyTest, DropTest) {
} }
} }
TEST_F(ColumnFamilyTest, WriteBatchFailure) { TEST_P(ColumnFamilyTest, WriteBatchFailure) {
Open(); Open();
CreateColumnFamiliesAndReopen({"one", "two"}); CreateColumnFamiliesAndReopen({"one", "two"});
WriteBatch batch; WriteBatch batch;
@ -806,7 +835,7 @@ TEST_F(ColumnFamilyTest, WriteBatchFailure) {
Close(); Close();
} }
TEST_F(ColumnFamilyTest, ReadWrite) { TEST_P(ColumnFamilyTest, ReadWrite) {
Open(); Open();
CreateColumnFamiliesAndReopen({"one", "two"}); CreateColumnFamiliesAndReopen({"one", "two"});
ASSERT_OK(Put(0, "foo", "v1")); ASSERT_OK(Put(0, "foo", "v1"));
@ -830,7 +859,7 @@ TEST_F(ColumnFamilyTest, ReadWrite) {
Close(); Close();
} }
TEST_F(ColumnFamilyTest, IgnoreRecoveredLog) { TEST_P(ColumnFamilyTest, IgnoreRecoveredLog) {
std::string backup_logs = dbname_ + "/backup_logs"; std::string backup_logs = dbname_ + "/backup_logs";
// delete old files in backup_logs directory // delete old files in backup_logs directory
@ -906,7 +935,7 @@ TEST_F(ColumnFamilyTest, IgnoreRecoveredLog) {
} }
#ifndef ROCKSDB_LITE // TEST functions used are not supported #ifndef ROCKSDB_LITE // TEST functions used are not supported
TEST_F(ColumnFamilyTest, FlushTest) { TEST_P(ColumnFamilyTest, FlushTest) {
Open(); Open();
CreateColumnFamiliesAndReopen({"one", "two"}); CreateColumnFamiliesAndReopen({"one", "two"});
ASSERT_OK(Put(0, "foo", "v1")); ASSERT_OK(Put(0, "foo", "v1"));
@ -954,7 +983,7 @@ TEST_F(ColumnFamilyTest, FlushTest) {
} }
// Makes sure that obsolete log files get deleted // Makes sure that obsolete log files get deleted
TEST_F(ColumnFamilyTest, LogDeletionTest) { TEST_P(ColumnFamilyTest, LogDeletionTest) {
db_options_.max_total_wal_size = std::numeric_limits<uint64_t>::max(); db_options_.max_total_wal_size = std::numeric_limits<uint64_t>::max();
column_family_options_.arena_block_size = 4 * 1024; column_family_options_.arena_block_size = 4 * 1024;
column_family_options_.write_buffer_size = 128000; // 128KB column_family_options_.write_buffer_size = 128000; // 128KB
@ -1022,7 +1051,7 @@ TEST_F(ColumnFamilyTest, LogDeletionTest) {
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
TEST_F(ColumnFamilyTest, CrashAfterFlush) { TEST_P(ColumnFamilyTest, CrashAfterFlush) {
std::unique_ptr<FaultInjectionTestEnv> fault_env( std::unique_ptr<FaultInjectionTestEnv> fault_env(
new FaultInjectionTestEnv(env_)); new FaultInjectionTestEnv(env_));
db_options_.env = fault_env.get(); db_options_.env = fault_env.get();
@ -1054,7 +1083,7 @@ TEST_F(ColumnFamilyTest, CrashAfterFlush) {
db_options_.env = env_; db_options_.env = env_;
} }
TEST_F(ColumnFamilyTest, OpenNonexistentColumnFamily) { TEST_P(ColumnFamilyTest, OpenNonexistentColumnFamily) {
ASSERT_OK(TryOpen({"default"})); ASSERT_OK(TryOpen({"default"}));
Close(); Close();
ASSERT_TRUE(TryOpen({"default", "dne"}).IsInvalidArgument()); ASSERT_TRUE(TryOpen({"default", "dne"}).IsInvalidArgument());
@ -1062,7 +1091,7 @@ TEST_F(ColumnFamilyTest, OpenNonexistentColumnFamily) {
#ifndef ROCKSDB_LITE // WaitForFlush() is not supported #ifndef ROCKSDB_LITE // WaitForFlush() is not supported
// Makes sure that obsolete log files get deleted // Makes sure that obsolete log files get deleted
TEST_F(ColumnFamilyTest, DifferentWriteBufferSizes) { TEST_P(ColumnFamilyTest, DifferentWriteBufferSizes) {
// disable flushing stale column families // disable flushing stale column families
db_options_.max_total_wal_size = std::numeric_limits<uint64_t>::max(); db_options_.max_total_wal_size = std::numeric_limits<uint64_t>::max();
Open(); Open();
@ -1168,7 +1197,7 @@ TEST_F(ColumnFamilyTest, DifferentWriteBufferSizes) {
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
#ifndef ROCKSDB_LITE // Cuckoo is not supported in lite #ifndef ROCKSDB_LITE // Cuckoo is not supported in lite
TEST_F(ColumnFamilyTest, MemtableNotSupportSnapshot) { TEST_P(ColumnFamilyTest, MemtableNotSupportSnapshot) {
db_options_.allow_concurrent_memtable_write = false; db_options_.allow_concurrent_memtable_write = false;
Open(); Open();
auto* s1 = dbfull()->GetSnapshot(); auto* s1 = dbfull()->GetSnapshot();
@ -1206,7 +1235,7 @@ static TestComparator third_comparator;
static TestComparator fourth_comparator; static TestComparator fourth_comparator;
// Test that we can retrieve the comparator from a created CF // Test that we can retrieve the comparator from a created CF
TEST_F(ColumnFamilyTest, GetComparator) { TEST_P(ColumnFamilyTest, GetComparator) {
Open(); Open();
// Add a column family with no comparator specified // Add a column family with no comparator specified
CreateColumnFamilies({"first"}); CreateColumnFamilies({"first"});
@ -1225,7 +1254,7 @@ TEST_F(ColumnFamilyTest, GetComparator) {
Close(); Close();
} }
TEST_F(ColumnFamilyTest, DifferentMergeOperators) { TEST_P(ColumnFamilyTest, DifferentMergeOperators) {
Open(); Open();
CreateColumnFamilies({"first", "second"}); CreateColumnFamilies({"first", "second"});
ColumnFamilyOptions default_cf, first, second; ColumnFamilyOptions default_cf, first, second;
@ -1256,7 +1285,7 @@ TEST_F(ColumnFamilyTest, DifferentMergeOperators) {
} }
#ifndef ROCKSDB_LITE // WaitForFlush() is not supported #ifndef ROCKSDB_LITE // WaitForFlush() is not supported
TEST_F(ColumnFamilyTest, DifferentCompactionStyles) { TEST_P(ColumnFamilyTest, DifferentCompactionStyles) {
Open(); Open();
CreateColumnFamilies({"one", "two"}); CreateColumnFamilies({"one", "two"});
ColumnFamilyOptions default_cf, one, two; ColumnFamilyOptions default_cf, one, two;
@ -1268,7 +1297,7 @@ TEST_F(ColumnFamilyTest, DifferentCompactionStyles) {
default_cf.target_file_size_base = 30 << 10; default_cf.target_file_size_base = 30 << 10;
default_cf.max_compaction_bytes = static_cast<uint64_t>(1) << 60; default_cf.max_compaction_bytes = static_cast<uint64_t>(1) << 60;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.no_block_cache = true; table_options.no_block_cache = true;
default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1328,7 +1357,7 @@ TEST_F(ColumnFamilyTest, DifferentCompactionStyles) {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
// Sync points not supported in RocksDB Lite // Sync points not supported in RocksDB Lite
TEST_F(ColumnFamilyTest, MultipleManualCompactions) { TEST_P(ColumnFamilyTest, MultipleManualCompactions) {
Open(); Open();
CreateColumnFamilies({"one", "two"}); CreateColumnFamilies({"one", "two"});
ColumnFamilyOptions default_cf, one, two; ColumnFamilyOptions default_cf, one, two;
@ -1340,7 +1369,7 @@ TEST_F(ColumnFamilyTest, MultipleManualCompactions) {
default_cf.write_buffer_size = 64 << 10; // 64KB default_cf.write_buffer_size = 64 << 10; // 64KB
default_cf.target_file_size_base = 30 << 10; default_cf.target_file_size_base = 30 << 10;
default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.no_block_cache = true; table_options.no_block_cache = true;
default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1422,10 +1451,12 @@ TEST_F(ColumnFamilyTest, MultipleManualCompactions) {
ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
key_iter++; key_iter++;
} }
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
Close(); Close();
} }
TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) { TEST_P(ColumnFamilyTest, AutomaticAndManualCompactions) {
Open(); Open();
CreateColumnFamilies({"one", "two"}); CreateColumnFamilies({"one", "two"});
ColumnFamilyOptions default_cf, one, two; ColumnFamilyOptions default_cf, one, two;
@ -1437,7 +1468,8 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
default_cf.write_buffer_size = 64 << 10; // 64KB default_cf.write_buffer_size = 64 << 10; // 64KB
default_cf.target_file_size_base = 30 << 10; default_cf.target_file_size_base = 30 << 10;
default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
;
table_options.no_block_cache = true; table_options.no_block_cache = true;
default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1516,9 +1548,11 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
key_iter++; key_iter++;
} }
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
} }
TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) { TEST_P(ColumnFamilyTest, ManualAndAutomaticCompactions) {
Open(); Open();
CreateColumnFamilies({"one", "two"}); CreateColumnFamilies({"one", "two"});
ColumnFamilyOptions default_cf, one, two; ColumnFamilyOptions default_cf, one, two;
@ -1530,7 +1564,8 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
default_cf.write_buffer_size = 64 << 10; // 64KB default_cf.write_buffer_size = 64 << 10; // 64KB
default_cf.target_file_size_base = 30 << 10; default_cf.target_file_size_base = 30 << 10;
default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
;
table_options.no_block_cache = true; table_options.no_block_cache = true;
default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1612,9 +1647,11 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
key_iter++; key_iter++;
} }
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
} }
TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) { TEST_P(ColumnFamilyTest, SameCFManualManualCompactions) {
Open(); Open();
CreateColumnFamilies({"one"}); CreateColumnFamilies({"one"});
ColumnFamilyOptions default_cf, one; ColumnFamilyOptions default_cf, one;
@ -1626,7 +1663,8 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
default_cf.write_buffer_size = 64 << 10; // 64KB default_cf.write_buffer_size = 64 << 10; // 64KB
default_cf.target_file_size_base = 30 << 10; default_cf.target_file_size_base = 30 << 10;
default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
;
table_options.no_block_cache = true; table_options.no_block_cache = true;
default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1711,9 +1749,11 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
key_iter++; key_iter++;
} }
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
} }
TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) { TEST_P(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
Open(); Open();
CreateColumnFamilies({"one"}); CreateColumnFamilies({"one"});
ColumnFamilyOptions default_cf, one; ColumnFamilyOptions default_cf, one;
@ -1725,7 +1765,8 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
default_cf.write_buffer_size = 64 << 10; // 64KB default_cf.write_buffer_size = 64 << 10; // 64KB
default_cf.target_file_size_base = 30 << 10; default_cf.target_file_size_base = 30 << 10;
default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
;
table_options.no_block_cache = true; table_options.no_block_cache = true;
default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1801,9 +1842,11 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
key_iter++; key_iter++;
} }
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
} }
TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) { TEST_P(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
Open(); Open();
CreateColumnFamilies({"one"}); CreateColumnFamilies({"one"});
ColumnFamilyOptions default_cf, one; ColumnFamilyOptions default_cf, one;
@ -1815,7 +1858,8 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
default_cf.write_buffer_size = 64 << 10; // 64KB default_cf.write_buffer_size = 64 << 10; // 64KB
default_cf.target_file_size_base = 30 << 10; default_cf.target_file_size_base = 30 << 10;
default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
;
table_options.no_block_cache = true; table_options.no_block_cache = true;
default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1891,6 +1935,8 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
key_iter++; key_iter++;
} }
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
} }
// In this test, we generate enough files to trigger automatic compactions. // In this test, we generate enough files to trigger automatic compactions.
@ -1899,7 +1945,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
// This will wait because the automatic compaction has files it needs. // This will wait because the automatic compaction has files it needs.
// Once the conflict is hit, the automatic compaction starts and ends // Once the conflict is hit, the automatic compaction starts and ends
// Then the manual will run and end. // Then the manual will run and end.
TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) { TEST_P(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
Open(); Open();
CreateColumnFamilies({"one"}); CreateColumnFamilies({"one"});
ColumnFamilyOptions default_cf, one; ColumnFamilyOptions default_cf, one;
@ -1911,7 +1957,8 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
default_cf.write_buffer_size = 64 << 10; // 64KB default_cf.write_buffer_size = 64 << 10; // 64KB
default_cf.target_file_size_base = 30 << 10; default_cf.target_file_size_base = 30 << 10;
default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100; default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
;
table_options.no_block_cache = true; table_options.no_block_cache = true;
default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options)); default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1980,6 +2027,8 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
ASSERT_NE("NOT_FOUND", Get(1, *key_iter)); ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
key_iter++; key_iter++;
} }
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
@ -1996,7 +2045,7 @@ std::string IterStatus(Iterator* iter) {
} }
} // anonymous namespace } // anonymous namespace
TEST_F(ColumnFamilyTest, NewIteratorsTest) { TEST_P(ColumnFamilyTest, NewIteratorsTest) {
// iter == 0 -- no tailing // iter == 0 -- no tailing
// iter == 2 -- tailing // iter == 2 -- tailing
for (int iter = 0; iter < 2; ++iter) { for (int iter = 0; iter < 2; ++iter) {
@ -2043,7 +2092,7 @@ TEST_F(ColumnFamilyTest, NewIteratorsTest) {
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
#ifndef ROCKSDB_LITE // ReadOnlyDB is not supported #ifndef ROCKSDB_LITE // ReadOnlyDB is not supported
TEST_F(ColumnFamilyTest, ReadOnlyDBTest) { TEST_P(ColumnFamilyTest, ReadOnlyDBTest) {
Open(); Open();
CreateColumnFamiliesAndReopen({"one", "two", "three", "four"}); CreateColumnFamiliesAndReopen({"one", "two", "three", "four"});
ASSERT_OK(Put(0, "a", "b")); ASSERT_OK(Put(0, "a", "b"));
@ -2095,7 +2144,7 @@ TEST_F(ColumnFamilyTest, ReadOnlyDBTest) {
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
#ifndef ROCKSDB_LITE // WaitForFlush() is not supported in lite #ifndef ROCKSDB_LITE // WaitForFlush() is not supported in lite
TEST_F(ColumnFamilyTest, DontRollEmptyLogs) { TEST_P(ColumnFamilyTest, DontRollEmptyLogs) {
Open(); Open();
CreateColumnFamiliesAndReopen({"one", "two", "three", "four"}); CreateColumnFamiliesAndReopen({"one", "two", "three", "four"});
@ -2119,7 +2168,7 @@ TEST_F(ColumnFamilyTest, DontRollEmptyLogs) {
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
#ifndef ROCKSDB_LITE // WaitForCompaction() is not supported in lite #ifndef ROCKSDB_LITE // WaitForCompaction() is not supported in lite
TEST_F(ColumnFamilyTest, FlushStaleColumnFamilies) { TEST_P(ColumnFamilyTest, FlushStaleColumnFamilies) {
Open(); Open();
CreateColumnFamilies({"one", "two"}); CreateColumnFamilies({"one", "two"});
ColumnFamilyOptions default_cf, one, two; ColumnFamilyOptions default_cf, one, two;
@ -2154,7 +2203,7 @@ TEST_F(ColumnFamilyTest, FlushStaleColumnFamilies) {
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
TEST_F(ColumnFamilyTest, CreateMissingColumnFamilies) { TEST_P(ColumnFamilyTest, CreateMissingColumnFamilies) {
Status s = TryOpen({"one", "two"}); Status s = TryOpen({"one", "two"});
ASSERT_TRUE(!s.ok()); ASSERT_TRUE(!s.ok());
db_options_.create_missing_column_families = true; db_options_.create_missing_column_families = true;
@ -2163,7 +2212,7 @@ TEST_F(ColumnFamilyTest, CreateMissingColumnFamilies) {
Close(); Close();
} }
TEST_F(ColumnFamilyTest, SanitizeOptions) { TEST_P(ColumnFamilyTest, SanitizeOptions) {
DBOptions db_options; DBOptions db_options;
for (int s = kCompactionStyleLevel; s <= kCompactionStyleUniversal; ++s) { for (int s = kCompactionStyleLevel; s <= kCompactionStyleUniversal; ++s) {
for (int l = 0; l <= 2; l++) { for (int l = 0; l <= 2; l++) {
@ -2212,7 +2261,7 @@ TEST_F(ColumnFamilyTest, SanitizeOptions) {
} }
} }
TEST_F(ColumnFamilyTest, ReadDroppedColumnFamily) { TEST_P(ColumnFamilyTest, ReadDroppedColumnFamily) {
// iter 0 -- drop CF, don't reopen // iter 0 -- drop CF, don't reopen
// iter 1 -- delete CF, reopen // iter 1 -- delete CF, reopen
for (int iter = 0; iter < 2; ++iter) { for (int iter = 0; iter < 2; ++iter) {
@ -2284,7 +2333,7 @@ TEST_F(ColumnFamilyTest, ReadDroppedColumnFamily) {
} }
} }
TEST_F(ColumnFamilyTest, FlushAndDropRaceCondition) { TEST_P(ColumnFamilyTest, FlushAndDropRaceCondition) {
db_options_.create_missing_column_families = true; db_options_.create_missing_column_families = true;
Open({"default", "one"}); Open({"default", "one"});
ColumnFamilyOptions options; ColumnFamilyOptions options;
@ -2350,12 +2399,13 @@ TEST_F(ColumnFamilyTest, FlushAndDropRaceCondition) {
// skipped as persisting options is not supported in ROCKSDB_LITE // skipped as persisting options is not supported in ROCKSDB_LITE
namespace { namespace {
std::atomic<int> test_stage(0); std::atomic<int> test_stage(0);
std::atomic<bool> ordered_by_writethread(false);
const int kMainThreadStartPersistingOptionsFile = 1; const int kMainThreadStartPersistingOptionsFile = 1;
const int kChildThreadFinishDroppingColumnFamily = 2; const int kChildThreadFinishDroppingColumnFamily = 2;
const int kChildThreadWaitingMainThreadPersistOptions = 3;
void DropSingleColumnFamily(ColumnFamilyTest* cf_test, int cf_id, void DropSingleColumnFamily(ColumnFamilyTest* cf_test, int cf_id,
std::vector<Comparator*>* comparators) { std::vector<Comparator*>* comparators) {
while (test_stage < kMainThreadStartPersistingOptionsFile) { while (test_stage < kMainThreadStartPersistingOptionsFile &&
!ordered_by_writethread) {
Env::Default()->SleepForMicroseconds(100); Env::Default()->SleepForMicroseconds(100);
} }
cf_test->DropColumnFamilies({cf_id}); cf_test->DropColumnFamilies({cf_id});
@ -2367,7 +2417,7 @@ void DropSingleColumnFamily(ColumnFamilyTest* cf_test, int cf_id,
} }
} // namespace } // namespace
TEST_F(ColumnFamilyTest, CreateAndDropRace) { TEST_P(ColumnFamilyTest, CreateAndDropRace) {
const int kCfCount = 5; const int kCfCount = 5;
std::vector<ColumnFamilyOptions> cf_opts; std::vector<ColumnFamilyOptions> cf_opts;
std::vector<Comparator*> comparators; std::vector<Comparator*> comparators;
@ -2388,7 +2438,8 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
// will be blocked until some other thread update the test_stage. // will be blocked until some other thread update the test_stage.
if (main_thread_id == current_thread_id) { if (main_thread_id == current_thread_id) {
test_stage = kMainThreadStartPersistingOptionsFile; test_stage = kMainThreadStartPersistingOptionsFile;
while (test_stage < kChildThreadFinishDroppingColumnFamily) { while (test_stage < kChildThreadFinishDroppingColumnFamily &&
!ordered_by_writethread) {
Env::Default()->SleepForMicroseconds(100); Env::Default()->SleepForMicroseconds(100);
} }
} }
@ -2399,11 +2450,7 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
// This means a thread doing DropColumnFamily() is waiting for // This means a thread doing DropColumnFamily() is waiting for
// other thread to finish persisting options. // other thread to finish persisting options.
// In such case, we update the test_stage to unblock the main thread. // In such case, we update the test_stage to unblock the main thread.
test_stage = kChildThreadWaitingMainThreadPersistOptions; ordered_by_writethread = true;
// Note that based on the test setting, this must not be the
// main thread.
ASSERT_NE(main_thread_id, std::this_thread::get_id());
}); });
// Create a database with four column families // Create a database with four column families
@ -2414,7 +2461,8 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
// Start a thread that will drop the first column family // Start a thread that will drop the first column family
// and its comparator // and its comparator
rocksdb::port::Thread drop_cf_thread(DropSingleColumnFamily, this, 1, &comparators); rocksdb::port::Thread drop_cf_thread(DropSingleColumnFamily, this, 1,
&comparators);
DropColumnFamilies({2}); DropColumnFamilies({2});
@ -2426,10 +2474,13 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
delete comparator; delete comparator;
} }
} }
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) { TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) {
const uint64_t kBaseRate = 800000u; const uint64_t kBaseRate = 800000u;
db_options_.delayed_write_rate = kBaseRate; db_options_.delayed_write_rate = kBaseRate;
db_options_.max_background_compactions = 6; db_options_.max_background_compactions = 6;
@ -2611,7 +2662,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
} }
TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) { TEST_P(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
db_options_.max_background_compactions = 6; db_options_.max_background_compactions = 6;
Open({"default"}); Open({"default"});
ColumnFamilyData* cfd = ColumnFamilyData* cfd =
@ -2675,7 +2726,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
} }
TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) { TEST_P(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
const uint64_t kBaseRate = 810000u; const uint64_t kBaseRate = 810000u;
db_options_.delayed_write_rate = kBaseRate; db_options_.delayed_write_rate = kBaseRate;
Open(); Open();
@ -2750,7 +2801,7 @@ TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
} }
TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) { TEST_P(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
db_options_.max_background_compactions = 6; db_options_.max_background_compactions = 6;
column_family_options_.soft_pending_compaction_bytes_limit = 200; column_family_options_.soft_pending_compaction_bytes_limit = 200;
column_family_options_.hard_pending_compaction_bytes_limit = 2000; column_family_options_.hard_pending_compaction_bytes_limit = 2000;
@ -2815,7 +2866,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
} }
TEST_F(ColumnFamilyTest, CreateAndDestoryOptions) { TEST_P(ColumnFamilyTest, CreateAndDestoryOptions) {
std::unique_ptr<ColumnFamilyOptions> cfo(new ColumnFamilyOptions()); std::unique_ptr<ColumnFamilyOptions> cfo(new ColumnFamilyOptions());
ColumnFamilyHandle* cfh; ColumnFamilyHandle* cfh;
Open(); Open();
@ -2827,7 +2878,7 @@ TEST_F(ColumnFamilyTest, CreateAndDestoryOptions) {
ASSERT_OK(db_->DestroyColumnFamilyHandle(cfh)); ASSERT_OK(db_->DestroyColumnFamilyHandle(cfh));
} }
TEST_F(ColumnFamilyTest, CreateDropAndDestroy) { TEST_P(ColumnFamilyTest, CreateDropAndDestroy) {
ColumnFamilyHandle* cfh; ColumnFamilyHandle* cfh;
Open(); Open();
ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(), "yoyo", &cfh)); ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(), "yoyo", &cfh));
@ -2838,7 +2889,7 @@ TEST_F(ColumnFamilyTest, CreateDropAndDestroy) {
} }
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
TEST_F(ColumnFamilyTest, CreateDropAndDestroyWithoutFileDeletion) { TEST_P(ColumnFamilyTest, CreateDropAndDestroyWithoutFileDeletion) {
ColumnFamilyHandle* cfh; ColumnFamilyHandle* cfh;
Open(); Open();
ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(), "yoyo", &cfh)); ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(), "yoyo", &cfh));
@ -2849,7 +2900,7 @@ TEST_F(ColumnFamilyTest, CreateDropAndDestroyWithoutFileDeletion) {
ASSERT_OK(db_->DestroyColumnFamilyHandle(cfh)); ASSERT_OK(db_->DestroyColumnFamilyHandle(cfh));
} }
TEST_F(ColumnFamilyTest, FlushCloseWALFiles) { TEST_P(ColumnFamilyTest, FlushCloseWALFiles) {
SpecialEnv env(Env::Default()); SpecialEnv env(Env::Default());
db_options_.env = &env; db_options_.env = &env;
db_options_.max_background_flushes = 1; db_options_.max_background_flushes = 1;
@ -2891,7 +2942,7 @@ TEST_F(ColumnFamilyTest, FlushCloseWALFiles) {
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
#ifndef ROCKSDB_LITE // WaitForFlush() is not supported #ifndef ROCKSDB_LITE // WaitForFlush() is not supported
TEST_F(ColumnFamilyTest, IteratorCloseWALFile1) { TEST_P(ColumnFamilyTest, IteratorCloseWALFile1) {
SpecialEnv env(Env::Default()); SpecialEnv env(Env::Default());
db_options_.env = &env; db_options_.env = &env;
db_options_.max_background_flushes = 1; db_options_.max_background_flushes = 1;
@ -2936,7 +2987,7 @@ TEST_F(ColumnFamilyTest, IteratorCloseWALFile1) {
Close(); Close();
} }
TEST_F(ColumnFamilyTest, IteratorCloseWALFile2) { TEST_P(ColumnFamilyTest, IteratorCloseWALFile2) {
SpecialEnv env(Env::Default()); SpecialEnv env(Env::Default());
// Allow both of flush and purge job to schedule. // Allow both of flush and purge job to schedule.
env.SetBackgroundThreads(2, Env::HIGH); env.SetBackgroundThreads(2, Env::HIGH);
@ -2993,7 +3044,7 @@ TEST_F(ColumnFamilyTest, IteratorCloseWALFile2) {
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
#ifndef ROCKSDB_LITE // TEST functions are not supported in lite #ifndef ROCKSDB_LITE // TEST functions are not supported in lite
TEST_F(ColumnFamilyTest, ForwardIteratorCloseWALFile) { TEST_P(ColumnFamilyTest, ForwardIteratorCloseWALFile) {
SpecialEnv env(Env::Default()); SpecialEnv env(Env::Default());
// Allow both of flush and purge job to schedule. // Allow both of flush and purge job to schedule.
env.SetBackgroundThreads(2, Env::HIGH); env.SetBackgroundThreads(2, Env::HIGH);
@ -3070,7 +3121,7 @@ TEST_F(ColumnFamilyTest, ForwardIteratorCloseWALFile) {
// Disable on windows because SyncWAL requires env->IsSyncThreadSafe() // Disable on windows because SyncWAL requires env->IsSyncThreadSafe()
// to return true which is not so in unbuffered mode. // to return true which is not so in unbuffered mode.
#ifndef OS_WIN #ifndef OS_WIN
TEST_F(ColumnFamilyTest, LogSyncConflictFlush) { TEST_P(ColumnFamilyTest, LogSyncConflictFlush) {
Open(); Open();
CreateColumnFamiliesAndReopen({"one", "two"}); CreateColumnFamiliesAndReopen({"one", "two"});
@ -3105,7 +3156,7 @@ TEST_F(ColumnFamilyTest, LogSyncConflictFlush) {
// test is being used to ensure a roll of wal files. // test is being used to ensure a roll of wal files.
// Basic idea is to test that WAL truncation is being detected and not // Basic idea is to test that WAL truncation is being detected and not
// ignored // ignored
TEST_F(ColumnFamilyTest, DISABLED_LogTruncationTest) { TEST_P(ColumnFamilyTest, DISABLED_LogTruncationTest) {
Open(); Open();
CreateColumnFamiliesAndReopen({"one", "two"}); CreateColumnFamiliesAndReopen({"one", "two"});
@ -3175,7 +3226,7 @@ TEST_F(ColumnFamilyTest, DISABLED_LogTruncationTest) {
env_->DeleteDir(backup_logs); env_->DeleteDir(backup_logs);
} }
TEST_F(ColumnFamilyTest, DefaultCfPathsTest) { TEST_P(ColumnFamilyTest, DefaultCfPathsTest) {
Open(); Open();
// Leave cf_paths for one column families to be empty. // Leave cf_paths for one column families to be empty.
// Files should be generated according to db_paths for that // Files should be generated according to db_paths for that
@ -3202,7 +3253,7 @@ TEST_F(ColumnFamilyTest, DefaultCfPathsTest) {
ASSERT_EQ(1, GetSstFileCount(dbname_)); ASSERT_EQ(1, GetSstFileCount(dbname_));
} }
TEST_F(ColumnFamilyTest, MultipleCFPathsTest) { TEST_P(ColumnFamilyTest, MultipleCFPathsTest) {
Open(); Open();
// Configure Column family specific paths. // Configure Column family specific paths.
ColumnFamilyOptions cf_opt1, cf_opt2; ColumnFamilyOptions cf_opt1, cf_opt2;

40
db/comparator_db_test.cc
Unescape View File

@ -251,18 +251,25 @@ class TwoStrComparator : public Comparator {
}; };
} // namespace } // namespace
class ComparatorDBTest : public testing::Test { class ComparatorDBTest
: public testing::Test,
virtual public ::testing::WithParamInterface<uint32_t> {
private: private:
std::string dbname_; std::string dbname_;
Env* env_; Env* env_;
DB* db_; DB* db_;
Options last_options_; Options last_options_;
std::unique_ptr<const Comparator> comparator_guard; std::unique_ptr<const Comparator> comparator_guard;
uint32_t format_;
public: public:
ComparatorDBTest() : env_(Env::Default()), db_(nullptr) { ComparatorDBTest() : env_(Env::Default()), db_(nullptr) {
comparator = BytewiseComparator(); comparator = BytewiseComparator();
dbname_ = test::TmpDir() + "/comparator_db_test"; dbname_ = test::TmpDir() + "/comparator_db_test";
BlockBasedTableOptions toptions;
toptions.format_version = GetParam();
last_options_.table_factory.reset(
rocksdb::NewBlockBasedTableFactory(toptions));
EXPECT_OK(DestroyDB(dbname_, last_options_)); EXPECT_OK(DestroyDB(dbname_, last_options_));
} }
@ -274,8 +281,12 @@ class ComparatorDBTest : public testing::Test {
DB* GetDB() { return db_; } DB* GetDB() { return db_; }
void SetOwnedComparator(const Comparator* cmp) { void SetOwnedComparator(const Comparator* cmp, bool owner = true) {
comparator_guard.reset(cmp); if (owner) {
comparator_guard.reset(cmp);
} else {
comparator_guard.reset();
}
comparator = cmp; comparator = cmp;
last_options_.comparator = cmp; last_options_.comparator = cmp;
} }
@ -304,7 +315,12 @@ class ComparatorDBTest : public testing::Test {
} }
}; };
TEST_F(ComparatorDBTest, Bytewise) { INSTANTIATE_TEST_CASE_P(FormatDef, ComparatorDBTest,
testing::Values(test::kDefaultFormatVersion));
INSTANTIATE_TEST_CASE_P(FormatLatest, ComparatorDBTest,
testing::Values(test::kLatestFormatVersion));
TEST_P(ComparatorDBTest, Bytewise) {
for (int rand_seed = 301; rand_seed < 306; rand_seed++) { for (int rand_seed = 301; rand_seed < 306; rand_seed++) {
DestroyAndReopen(); DestroyAndReopen();
Random rnd(rand_seed); Random rnd(rand_seed);
@ -314,7 +330,7 @@ TEST_F(ComparatorDBTest, Bytewise) {
} }
} }
TEST_F(ComparatorDBTest, SimpleSuffixReverseComparator) { TEST_P(ComparatorDBTest, SimpleSuffixReverseComparator) {
SetOwnedComparator(new test::SimpleSuffixReverseComparator()); SetOwnedComparator(new test::SimpleSuffixReverseComparator());
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
@ -340,8 +356,8 @@ TEST_F(ComparatorDBTest, SimpleSuffixReverseComparator) {
} }
} }
TEST_F(ComparatorDBTest, Uint64Comparator) { TEST_P(ComparatorDBTest, Uint64Comparator) {
SetOwnedComparator(test::Uint64Comparator()); SetOwnedComparator(test::Uint64Comparator(), false /* owner */);
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
Options* opt = GetOptions(); Options* opt = GetOptions();
@ -364,7 +380,7 @@ TEST_F(ComparatorDBTest, Uint64Comparator) {
} }
} }
TEST_F(ComparatorDBTest, DoubleComparator) { TEST_P(ComparatorDBTest, DoubleComparator) {
SetOwnedComparator(new DoubleComparator()); SetOwnedComparator(new DoubleComparator());
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
@ -389,7 +405,7 @@ TEST_F(ComparatorDBTest, DoubleComparator) {
} }
} }
TEST_F(ComparatorDBTest, HashComparator) { TEST_P(ComparatorDBTest, HashComparator) {
SetOwnedComparator(new HashComparator()); SetOwnedComparator(new HashComparator());
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
@ -408,7 +424,7 @@ TEST_F(ComparatorDBTest, HashComparator) {
} }
} }
TEST_F(ComparatorDBTest, TwoStrComparator) { TEST_P(ComparatorDBTest, TwoStrComparator) {
SetOwnedComparator(new TwoStrComparator()); SetOwnedComparator(new TwoStrComparator());
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
@ -434,7 +450,7 @@ TEST_F(ComparatorDBTest, TwoStrComparator) {
} }
} }
TEST_F(ComparatorDBTest, FindShortestSeparator) { TEST_P(ComparatorDBTest, FindShortestSeparator) {
std::string s1 = "abc1xyz"; std::string s1 = "abc1xyz";
std::string s2 = "abc3xy"; std::string s2 = "abc3xy";
@ -468,7 +484,7 @@ TEST_F(ComparatorDBTest, FindShortestSeparator) {
ASSERT_TRUE(s1 > s2); ASSERT_TRUE(s1 > s2);
} }
TEST_F(ComparatorDBTest, SeparatorSuccessorRandomizeTest) { TEST_P(ComparatorDBTest, SeparatorSuccessorRandomizeTest) {
// Char list for boundary cases. // Char list for boundary cases.
std::array<unsigned char, 6> char_list{{0, 1, 2, 253, 254, 255}}; std::array<unsigned char, 6> char_list{{0, 1, 2, 253, 254, 255}};
Random rnd(301); Random rnd(301);

9
db/db_test_util.cc
Unescape View File

@ -448,6 +448,15 @@ Options DBTestBase::GetOptions(
options.prefix_extractor.reset(NewNoopTransform()); options.prefix_extractor.reset(NewNoopTransform());
break; break;
} }
case kBlockBasedTableWithPartitionedIndexFormat3: {
table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch;
options.prefix_extractor.reset(NewNoopTransform());
// Format 3 changes the binary index format. Since partitioned index is a
// super-set of simple indexes, we are also using kTwoLevelIndexSearch to
// test this format.
table_options.format_version = 3;
break;
}
case kBlockBasedTableWithIndexRestartInterval: { case kBlockBasedTableWithIndexRestartInterval: {
table_options.index_block_restart_interval = 8; table_options.index_block_restart_interval = 8;
break; break;

3
db/db_test_util.h
Unescape View File

@ -698,7 +698,10 @@ class DBTestBase : public testing::Test {
kLevelSubcompactions, kLevelSubcompactions,
kBlockBasedTableWithIndexRestartInterval, kBlockBasedTableWithIndexRestartInterval,
kBlockBasedTableWithPartitionedIndex, kBlockBasedTableWithPartitionedIndex,
kBlockBasedTableWithPartitionedIndexFormat3,
kPartitionedFilterWithNewTableReaderForCompactions, kPartitionedFilterWithNewTableReaderForCompactions,
// This must be the last line
kEnd, kEnd,
// TODO: This option although been there for a while was disable due to a // TODO: This option although been there for a while was disable due to a

15
table/block_based_table_reader.cc
Unescape View File

@ -176,9 +176,8 @@ Cache::Handle* GetEntryFromCache(Cache* block_cache, const Slice& key,
// For hash based index, return true if prefix_extractor and // For hash based index, return true if prefix_extractor and
// prefix_extractor_block mismatch, false otherwise. This flag will be used // prefix_extractor_block mismatch, false otherwise. This flag will be used
// as total_order_seek via NewIndexIterator // as total_order_seek via NewIndexIterator
bool PrefixExtractorChanged( bool PrefixExtractorChanged(const TableProperties* table_properties,
const TableProperties* table_properties, const SliceTransform* prefix_extractor) {
const SliceTransform* prefix_extractor) {
// BlockBasedTableOptions::kHashSearch requires prefix_extractor to be set. // BlockBasedTableOptions::kHashSearch requires prefix_extractor to be set.
// Turn off hash index in prefix_extractor is not set; if prefix_extractor // Turn off hash index in prefix_extractor is not set; if prefix_extractor
// is set but prefix_extractor_block is not set, also disable hash index // is set but prefix_extractor_block is not set, also disable hash index
@ -912,9 +911,8 @@ Status BlockBasedTable::Open(const ImmutableCFOptions& ioptions,
bool prefix_extractor_changed = false; bool prefix_extractor_changed = false;
// check prefix_extractor match only if hash based index is used // check prefix_extractor match only if hash based index is used
if (rep->index_type == BlockBasedTableOptions::kHashSearch) { if (rep->index_type == BlockBasedTableOptions::kHashSearch) {
prefix_extractor_changed = prefix_extractor_changed = PrefixExtractorChanged(
PrefixExtractorChanged(rep->table_properties.get(), rep->table_properties.get(), prefix_extractor);
prefix_extractor);
} }
unique_ptr<InternalIterator> iter(new_table->NewIndexIterator( unique_ptr<InternalIterator> iter(new_table->NewIndexIterator(
ReadOptions(), prefix_extractor_changed, nullptr, &index_entry)); ReadOptions(), prefix_extractor_changed, nullptr, &index_entry));
@ -2176,9 +2174,8 @@ Status BlockBasedTable::Get(const ReadOptions& read_options, const Slice& key,
// BlockPrefixIndex. Only do this check when index_type is kHashSearch. // BlockPrefixIndex. Only do this check when index_type is kHashSearch.
bool prefix_extractor_changed = false; bool prefix_extractor_changed = false;
if (rep_->index_type == BlockBasedTableOptions::kHashSearch) { if (rep_->index_type == BlockBasedTableOptions::kHashSearch) {
prefix_extractor_changed = prefix_extractor_changed = PrefixExtractorChanged(
PrefixExtractorChanged(rep_->table_properties.get(), rep_->table_properties.get(), prefix_extractor);
prefix_extractor);
} }
auto iiter = NewIndexIterator(read_options, prefix_extractor_changed, auto iiter = NewIndexIterator(read_options, prefix_extractor_changed,
&iiter_on_stack, /* index_entry */ nullptr, &iiter_on_stack, /* index_entry */ nullptr,

2
table/format.h
Unescape View File

@ -83,7 +83,7 @@ inline uint32_t GetCompressFormatForVersion(
} }
inline bool BlockBasedTableSupportedVersion(uint32_t version) { inline bool BlockBasedTableSupportedVersion(uint32_t version) {
return version <= 2; return version <= 3;
} }
// Footer encapsulates the fixed information stored at the tail // Footer encapsulates the fixed information stored at the tail

119
table/table_test.cc
Unescape View File

@ -1049,13 +1049,32 @@ class TableTest : public testing::Test {
}; };
class GeneralTableTest : public TableTest {}; class GeneralTableTest : public TableTest {};
class BlockBasedTableTest : public TableTest { class BlockBasedTableTest
: public TableTest,
virtual public ::testing::WithParamInterface<uint32_t> {
public:
BlockBasedTableTest() : format_(GetParam()) {}
BlockBasedTableOptions GetBlockBasedTableOptions() {
BlockBasedTableOptions options;
options.format_version = format_;
return options;
}
protected: protected:
uint64_t IndexUncompressedHelper(bool indexCompress); uint64_t IndexUncompressedHelper(bool indexCompress);
private:
uint32_t format_;
}; };
class PlainTableTest : public TableTest {}; class PlainTableTest : public TableTest {};
class TablePropertyTest : public testing::Test {}; class TablePropertyTest : public testing::Test {};
INSTANTIATE_TEST_CASE_P(FormatDef, BlockBasedTableTest,
testing::Values(test::kDefaultFormatVersion));
INSTANTIATE_TEST_CASE_P(FormatLatest, BlockBasedTableTest,
testing::Values(test::kLatestFormatVersion));
// This test serves as the living tutorial for the prefix scan of user collected // This test serves as the living tutorial for the prefix scan of user collected
// properties. // properties.
TEST_F(TablePropertyTest, PrefixScanTest) { TEST_F(TablePropertyTest, PrefixScanTest) {
@ -1095,7 +1114,7 @@ TEST_F(TablePropertyTest, PrefixScanTest) {
// This test include all the basic checks except those for index size and block // This test include all the basic checks except those for index size and block
// size, which will be conducted in separated unit tests. // size, which will be conducted in separated unit tests.
TEST_F(BlockBasedTableTest, BasicBlockBasedTableProperties) { TEST_P(BlockBasedTableTest, BasicBlockBasedTableProperties) {
TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
c.Add("a1", "val1"); c.Add("a1", "val1");
@ -1115,7 +1134,7 @@ TEST_F(BlockBasedTableTest, BasicBlockBasedTableProperties) {
options.compression = kNoCompression; options.compression = kNoCompression;
options.statistics = CreateDBStatistics(); options.statistics = CreateDBStatistics();
options.statistics->stats_level_ = StatsLevel::kAll; options.statistics->stats_level_ = StatsLevel::kAll;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_restart_interval = 1; table_options.block_restart_interval = 1;
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1163,7 +1182,7 @@ uint64_t BlockBasedTableTest::IndexUncompressedHelper(bool compressed) {
options.compression = kSnappyCompression; options.compression = kSnappyCompression;
options.statistics = CreateDBStatistics(); options.statistics = CreateDBStatistics();
options.statistics->stats_level_ = StatsLevel::kAll; options.statistics->stats_level_ = StatsLevel::kAll;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_restart_interval = 1; table_options.block_restart_interval = 1;
table_options.enable_index_compression = compressed; table_options.enable_index_compression = compressed;
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1176,7 +1195,7 @@ uint64_t BlockBasedTableTest::IndexUncompressedHelper(bool compressed) {
c.ResetTableReader(); c.ResetTableReader();
return options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED); return options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
} }
TEST_F(BlockBasedTableTest, IndexUncompressed) { TEST_P(BlockBasedTableTest, IndexUncompressed) {
uint64_t tbl1_compressed_cnt = IndexUncompressedHelper(true); uint64_t tbl1_compressed_cnt = IndexUncompressedHelper(true);
uint64_t tbl2_compressed_cnt = IndexUncompressedHelper(false); uint64_t tbl2_compressed_cnt = IndexUncompressedHelper(false);
// tbl1_compressed_cnt should include 1 index block // tbl1_compressed_cnt should include 1 index block
@ -1184,7 +1203,7 @@ TEST_F(BlockBasedTableTest, IndexUncompressed) {
} }
#endif // SNAPPY #endif // SNAPPY
TEST_F(BlockBasedTableTest, BlockBasedTableProperties2) { TEST_P(BlockBasedTableTest, BlockBasedTableProperties2) {
TableConstructor c(&reverse_key_comparator); TableConstructor c(&reverse_key_comparator);
std::vector<std::string> keys; std::vector<std::string> keys;
stl_wrappers::KVMap kvmap; stl_wrappers::KVMap kvmap;
@ -1192,7 +1211,7 @@ TEST_F(BlockBasedTableTest, BlockBasedTableProperties2) {
{ {
Options options; Options options;
options.compression = CompressionType::kNoCompression; options.compression = CompressionType::kNoCompression;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions(options); const ImmutableCFOptions ioptions(options);
@ -1219,7 +1238,7 @@ TEST_F(BlockBasedTableTest, BlockBasedTableProperties2) {
{ {
Options options; Options options;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
options.comparator = &reverse_key_comparator; options.comparator = &reverse_key_comparator;
options.merge_operator = MergeOperators::CreateUInt64AddOperator(); options.merge_operator = MergeOperators::CreateUInt64AddOperator();
@ -1246,7 +1265,7 @@ TEST_F(BlockBasedTableTest, BlockBasedTableProperties2) {
} }
} }
TEST_F(BlockBasedTableTest, RangeDelBlock) { TEST_P(BlockBasedTableTest, RangeDelBlock) {
TableConstructor c(BytewiseComparator()); TableConstructor c(BytewiseComparator());
std::vector<std::string> keys = {"1pika", "2chu"}; std::vector<std::string> keys = {"1pika", "2chu"};
std::vector<std::string> vals = {"p", "c"}; std::vector<std::string> vals = {"p", "c"};
@ -1261,7 +1280,7 @@ TEST_F(BlockBasedTableTest, RangeDelBlock) {
stl_wrappers::KVMap kvmap; stl_wrappers::KVMap kvmap;
Options options; Options options;
options.compression = kNoCompression; options.compression = kNoCompression;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_restart_interval = 1; table_options.block_restart_interval = 1;
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1297,12 +1316,12 @@ TEST_F(BlockBasedTableTest, RangeDelBlock) {
} }
} }
TEST_F(BlockBasedTableTest, FilterPolicyNameProperties) { TEST_P(BlockBasedTableTest, FilterPolicyNameProperties) {
TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
c.Add("a1", "val1"); c.Add("a1", "val1");
std::vector<std::string> keys; std::vector<std::string> keys;
stl_wrappers::KVMap kvmap; stl_wrappers::KVMap kvmap;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.filter_policy.reset(NewBloomFilterPolicy(10)); table_options.filter_policy.reset(NewBloomFilterPolicy(10));
Options options; Options options;
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1386,14 +1405,14 @@ void PrefetchRange(TableConstructor* c, Options* opt,
c->ResetTableReader(); c->ResetTableReader();
} }
TEST_F(BlockBasedTableTest, PrefetchTest) { TEST_P(BlockBasedTableTest, PrefetchTest) {
// The purpose of this test is to test the prefetching operation built into // The purpose of this test is to test the prefetching operation built into
// BlockBasedTable. // BlockBasedTable.
Options opt; Options opt;
unique_ptr<InternalKeyComparator> ikc; unique_ptr<InternalKeyComparator> ikc;
ikc.reset(new test::PlainInternalKeyComparator(opt.comparator)); ikc.reset(new test::PlainInternalKeyComparator(opt.comparator));
opt.compression = kNoCompression; opt.compression = kNoCompression;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_size = 1024; table_options.block_size = 1024;
// big enough so we don't ever lose cached values. // big enough so we don't ever lose cached values.
table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4); table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4);
@ -1454,8 +1473,8 @@ TEST_F(BlockBasedTableTest, PrefetchTest) {
c.ResetTableReader(); c.ResetTableReader();
} }
TEST_F(BlockBasedTableTest, TotalOrderSeekOnHashIndex) { TEST_P(BlockBasedTableTest, TotalOrderSeekOnHashIndex) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
for (int i = 0; i < 4; ++i) { for (int i = 0; i < 4; ++i) {
Options options; Options options;
// Make each key/value an individual block // Make each key/value an individual block
@ -1546,8 +1565,8 @@ TEST_F(BlockBasedTableTest, TotalOrderSeekOnHashIndex) {
} }
} }
TEST_F(BlockBasedTableTest, NoopTransformSeek) { TEST_P(BlockBasedTableTest, NoopTransformSeek) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.filter_policy.reset(NewBloomFilterPolicy(10)); table_options.filter_policy.reset(NewBloomFilterPolicy(10));
Options options; Options options;
@ -1583,10 +1602,10 @@ TEST_F(BlockBasedTableTest, NoopTransformSeek) {
} }
} }
TEST_F(BlockBasedTableTest, SkipPrefixBloomFilter) { TEST_P(BlockBasedTableTest, SkipPrefixBloomFilter) {
// if DB is opened with a prefix extractor of a different name, // if DB is opened with a prefix extractor of a different name,
// prefix bloom is skipped when read the file // prefix bloom is skipped when read the file
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.filter_policy.reset(NewBloomFilterPolicy(2)); table_options.filter_policy.reset(NewBloomFilterPolicy(2));
table_options.whole_key_filtering = false; table_options.whole_key_filtering = false;
@ -1750,24 +1769,24 @@ void TableTest::IndexTest(BlockBasedTableOptions table_options) {
c.ResetTableReader(); c.ResetTableReader();
} }
TEST_F(TableTest, BinaryIndexTest) { TEST_P(BlockBasedTableTest, BinaryIndexTest) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.index_type = BlockBasedTableOptions::kBinarySearch; table_options.index_type = BlockBasedTableOptions::kBinarySearch;
IndexTest(table_options); IndexTest(table_options);
} }
TEST_F(TableTest, HashIndexTest) { TEST_P(BlockBasedTableTest, HashIndexTest) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.index_type = BlockBasedTableOptions::kHashSearch; table_options.index_type = BlockBasedTableOptions::kHashSearch;
IndexTest(table_options); IndexTest(table_options);
} }
TEST_F(TableTest, PartitionIndexTest) { TEST_P(BlockBasedTableTest, PartitionIndexTest) {
const int max_index_keys = 5; const int max_index_keys = 5;
const int est_max_index_key_value_size = 32; const int est_max_index_key_value_size = 32;
const int est_max_index_size = max_index_keys * est_max_index_key_value_size; const int est_max_index_size = max_index_keys * est_max_index_key_value_size;
for (int i = 1; i <= est_max_index_size + 1; i++) { for (int i = 1; i <= est_max_index_size + 1; i++) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch; table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch;
table_options.metadata_block_size = i; table_options.metadata_block_size = i;
IndexTest(table_options); IndexTest(table_options);
@ -1777,7 +1796,7 @@ TEST_F(TableTest, PartitionIndexTest) {
// It's very hard to figure out the index block size of a block accurately. // It's very hard to figure out the index block size of a block accurately.
// To make sure we get the index size, we just make sure as key number // To make sure we get the index size, we just make sure as key number
// grows, the filter block size also grows. // grows, the filter block size also grows.
TEST_F(BlockBasedTableTest, IndexSizeStat) { TEST_P(BlockBasedTableTest, IndexSizeStat) {
uint64_t last_index_size = 0; uint64_t last_index_size = 0;
// we need to use random keys since the pure human readable texts // we need to use random keys since the pure human readable texts
@ -1803,7 +1822,7 @@ TEST_F(BlockBasedTableTest, IndexSizeStat) {
stl_wrappers::KVMap kvmap; stl_wrappers::KVMap kvmap;
Options options; Options options;
options.compression = kNoCompression; options.compression = kNoCompression;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_restart_interval = 1; table_options.block_restart_interval = 1;
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1818,12 +1837,12 @@ TEST_F(BlockBasedTableTest, IndexSizeStat) {
} }
} }
TEST_F(BlockBasedTableTest, NumBlockStat) { TEST_P(BlockBasedTableTest, NumBlockStat) {
Random rnd(test::RandomSeed()); Random rnd(test::RandomSeed());
TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
Options options; Options options;
options.compression = kNoCompression; options.compression = kNoCompression;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_restart_interval = 1; table_options.block_restart_interval = 1;
table_options.block_size = 1000; table_options.block_size = 1000;
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1910,11 +1929,11 @@ class BlockCachePropertiesSnapshot {
// Make sure, by default, index/filter blocks were pre-loaded (meaning we won't // Make sure, by default, index/filter blocks were pre-loaded (meaning we won't
// use block cache to store them). // use block cache to store them).
TEST_F(BlockBasedTableTest, BlockCacheDisabledTest) { TEST_P(BlockBasedTableTest, BlockCacheDisabledTest) {
Options options; Options options;
options.create_if_missing = true; options.create_if_missing = true;
options.statistics = CreateDBStatistics(); options.statistics = CreateDBStatistics();
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_cache = NewLRUCache(1024, 4); table_options.block_cache = NewLRUCache(1024, 4);
table_options.filter_policy.reset(NewBloomFilterPolicy(10)); table_options.filter_policy.reset(NewBloomFilterPolicy(10));
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(new BlockBasedTableFactory(table_options));
@ -1955,14 +1974,14 @@ TEST_F(BlockBasedTableTest, BlockCacheDisabledTest) {
// Due to the difficulities of the intersaction between statistics, this test // Due to the difficulities of the intersaction between statistics, this test
// only tests the case when "index block is put to block cache" // only tests the case when "index block is put to block cache"
TEST_F(BlockBasedTableTest, FilterBlockInBlockCache) { TEST_P(BlockBasedTableTest, FilterBlockInBlockCache) {
// -- Table construction // -- Table construction
Options options; Options options;
options.create_if_missing = true; options.create_if_missing = true;
options.statistics = CreateDBStatistics(); options.statistics = CreateDBStatistics();
// Enable the cache for index/filter blocks // Enable the cache for index/filter blocks
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_cache = NewLRUCache(1024, 4); table_options.block_cache = NewLRUCache(1024, 4);
table_options.cache_index_and_filter_blocks = true; table_options.cache_index_and_filter_blocks = true;
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(new BlockBasedTableFactory(table_options));
@ -2147,7 +2166,7 @@ void ValidateBlockRestartInterval(int value, int expected) {
delete factory; delete factory;
} }
TEST_F(BlockBasedTableTest, InvalidOptions) { TEST_P(BlockBasedTableTest, InvalidOptions) {
// invalid values for block_size_deviation (<0 or >100) are silently set to 0 // invalid values for block_size_deviation (<0 or >100) are silently set to 0
ValidateBlockSizeDeviation(-10, 0); ValidateBlockSizeDeviation(-10, 0);
ValidateBlockSizeDeviation(-1, 0); ValidateBlockSizeDeviation(-1, 0);
@ -2167,7 +2186,7 @@ TEST_F(BlockBasedTableTest, InvalidOptions) {
ValidateBlockRestartInterval(1000, 1000); ValidateBlockRestartInterval(1000, 1000);
} }
TEST_F(BlockBasedTableTest, BlockReadCountTest) { TEST_P(BlockBasedTableTest, BlockReadCountTest) {
// bloom_filter_type = 0 -- block-based filter // bloom_filter_type = 0 -- block-based filter
// bloom_filter_type = 0 -- full filter // bloom_filter_type = 0 -- full filter
for (int bloom_filter_type = 0; bloom_filter_type < 2; ++bloom_filter_type) { for (int bloom_filter_type = 0; bloom_filter_type < 2; ++bloom_filter_type) {
@ -2176,7 +2195,7 @@ TEST_F(BlockBasedTableTest, BlockReadCountTest) {
Options options; Options options;
options.create_if_missing = true; options.create_if_missing = true;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_cache = NewLRUCache(1, 0); table_options.block_cache = NewLRUCache(1, 0);
table_options.cache_index_and_filter_blocks = index_and_filter_in_cache; table_options.cache_index_and_filter_blocks = index_and_filter_in_cache;
table_options.filter_policy.reset( table_options.filter_policy.reset(
@ -2295,7 +2314,7 @@ std::map<std::string, size_t> MockCache::marked_data_in_cache_;
// object depends on the table to be live, it then must be destructed before the // object depends on the table to be live, it then must be destructed before the
// table is closed. This test makes sure that the only items remains in the // table is closed. This test makes sure that the only items remains in the
// cache after the table is closed are raw data blocks. // cache after the table is closed are raw data blocks.
TEST_F(BlockBasedTableTest, NoObjectInCacheAfterTableClose) { TEST_P(BlockBasedTableTest, NoObjectInCacheAfterTableClose) {
for (int level: {-1, 0, 1, 10}) { for (int level: {-1, 0, 1, 10}) {
for (auto index_type : for (auto index_type :
{BlockBasedTableOptions::IndexType::kBinarySearch, {BlockBasedTableOptions::IndexType::kBinarySearch,
@ -2318,7 +2337,7 @@ TEST_F(BlockBasedTableTest, NoObjectInCacheAfterTableClose) {
unique_ptr<InternalKeyComparator> ikc; unique_ptr<InternalKeyComparator> ikc;
ikc.reset(new test::PlainInternalKeyComparator(opt.comparator)); ikc.reset(new test::PlainInternalKeyComparator(opt.comparator));
opt.compression = kNoCompression; opt.compression = kNoCompression;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_size = 1024; table_options.block_size = 1024;
table_options.index_type = table_options.index_type =
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
@ -2382,7 +2401,7 @@ TEST_F(BlockBasedTableTest, NoObjectInCacheAfterTableClose) {
} // level } // level
} }
TEST_F(BlockBasedTableTest, BlockCacheLeak) { TEST_P(BlockBasedTableTest, BlockCacheLeak) {
// Check that when we reopen a table we don't lose access to blocks already // Check that when we reopen a table we don't lose access to blocks already
// in the cache. This test checks whether the Table actually makes use of the // in the cache. This test checks whether the Table actually makes use of the
// unique ID from the file. // unique ID from the file.
@ -2391,7 +2410,7 @@ TEST_F(BlockBasedTableTest, BlockCacheLeak) {
unique_ptr<InternalKeyComparator> ikc; unique_ptr<InternalKeyComparator> ikc;
ikc.reset(new test::PlainInternalKeyComparator(opt.comparator)); ikc.reset(new test::PlainInternalKeyComparator(opt.comparator));
opt.compression = kNoCompression; opt.compression = kNoCompression;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.block_size = 1024; table_options.block_size = 1024;
// big enough so we don't ever lose cached values. // big enough so we don't ever lose cached values.
table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4); table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4);
@ -2446,7 +2465,7 @@ TEST_F(BlockBasedTableTest, BlockCacheLeak) {
c.ResetTableReader(); c.ResetTableReader();
} }
TEST_F(BlockBasedTableTest, NewIndexIteratorLeak) { TEST_P(BlockBasedTableTest, NewIndexIteratorLeak) {
// A regression test to avoid data race described in // A regression test to avoid data race described in
// https://github.com/facebook/rocksdb/issues/1267 // https://github.com/facebook/rocksdb/issues/1267
TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */); TableConstructor c(BytewiseComparator(), true /* convert_to_internal_key_ */);
@ -2455,7 +2474,7 @@ TEST_F(BlockBasedTableTest, NewIndexIteratorLeak) {
c.Add("a1", "val1"); c.Add("a1", "val1");
Options options; Options options;
options.prefix_extractor.reset(NewFixedPrefixTransform(1)); options.prefix_extractor.reset(NewFixedPrefixTransform(1));
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.index_type = BlockBasedTableOptions::kHashSearch; table_options.index_type = BlockBasedTableOptions::kHashSearch;
table_options.cache_index_and_filter_blocks = true; table_options.cache_index_and_filter_blocks = true;
table_options.block_cache = NewLRUCache(0); table_options.block_cache = NewLRUCache(0);
@ -2957,7 +2976,7 @@ TEST_F(HarnessTest, FooterTests) {
} }
class IndexBlockRestartIntervalTest class IndexBlockRestartIntervalTest
: public BlockBasedTableTest, : public TableTest,
public ::testing::WithParamInterface<int> { public ::testing::WithParamInterface<int> {
public: public:
static std::vector<int> GetRestartValues() { return {-1, 0, 1, 8, 16, 32}; } static std::vector<int> GetRestartValues() { return {-1, 0, 1, 8, 16, 32}; }
@ -3102,8 +3121,8 @@ TEST_F(PrefixTest, PrefixAndWholeKeyTest) {
// rocksdb still works. // rocksdb still works.
} }
TEST_F(BlockBasedTableTest, TableWithGlobalSeqno) { TEST_P(BlockBasedTableTest, TableWithGlobalSeqno) {
BlockBasedTableOptions bbto; BlockBasedTableOptions bbto = GetBlockBasedTableOptions();
test::StringSink* sink = new test::StringSink(); test::StringSink* sink = new test::StringSink();
unique_ptr<WritableFileWriter> file_writer(test::GetWritableFileWriter(sink)); unique_ptr<WritableFileWriter> file_writer(test::GetWritableFileWriter(sink));
Options options; Options options;
@ -3282,8 +3301,8 @@ TEST_F(BlockBasedTableTest, TableWithGlobalSeqno) {
delete iter; delete iter;
} }
TEST_F(BlockBasedTableTest, BlockAlignTest) { TEST_P(BlockBasedTableTest, BlockAlignTest) {
BlockBasedTableOptions bbto; BlockBasedTableOptions bbto = GetBlockBasedTableOptions();
bbto.block_align = true; bbto.block_align = true;
test::StringSink* sink = new test::StringSink(); test::StringSink* sink = new test::StringSink();
unique_ptr<WritableFileWriter> file_writer(test::GetWritableFileWriter(sink)); unique_ptr<WritableFileWriter> file_writer(test::GetWritableFileWriter(sink));
@ -3371,10 +3390,10 @@ TEST_F(BlockBasedTableTest, BlockAlignTest) {
table_reader.reset(); table_reader.reset();
} }
TEST_F(BlockBasedTableTest, BadOptions) { TEST_P(BlockBasedTableTest, BadOptions) {
rocksdb::Options options; rocksdb::Options options;
options.compression = kNoCompression; options.compression = kNoCompression;
rocksdb::BlockBasedTableOptions bbto; BlockBasedTableOptions bbto = GetBlockBasedTableOptions();
bbto.block_size = 4000; bbto.block_size = 4000;
bbto.block_align = true; bbto.block_align = true;

8
tools/db_bench_tool.cc
Unescape View File

@ -436,6 +436,11 @@ DEFINE_int32(block_size,
static_cast<int32_t>(rocksdb::BlockBasedTableOptions().block_size), static_cast<int32_t>(rocksdb::BlockBasedTableOptions().block_size),
"Number of bytes in a block."); "Number of bytes in a block.");
DEFINE_int32(
format_version,
static_cast<int32_t>(rocksdb::BlockBasedTableOptions().format_version),
"Format version of SST files.");
DEFINE_int32(block_restart_interval, DEFINE_int32(block_restart_interval,
rocksdb::BlockBasedTableOptions().block_restart_interval, rocksdb::BlockBasedTableOptions().block_restart_interval,
"Number of keys between restart points " "Number of keys between restart points "
@ -3161,7 +3166,8 @@ void VerifyDBFromDB(std::string& truth_db_name) {
block_based_options.index_block_restart_interval = block_based_options.index_block_restart_interval =
FLAGS_index_block_restart_interval; FLAGS_index_block_restart_interval;
block_based_options.filter_policy = filter_policy_; block_based_options.filter_policy = filter_policy_;
block_based_options.format_version = 2; block_based_options.format_version =
static_cast<uint32_t>(FLAGS_format_version);
block_based_options.read_amp_bytes_per_bit = FLAGS_read_amp_bytes_per_bit; block_based_options.read_amp_bytes_per_bit = FLAGS_read_amp_bytes_per_bit;
block_based_options.enable_index_compression = block_based_options.enable_index_compression =
FLAGS_enable_index_compression; FLAGS_enable_index_compression;

1
tools/db_crashtest.py
Unescape View File

@ -50,6 +50,7 @@ default_params = {
"verify_checksum": 1, "verify_checksum": 1,
"write_buffer_size": 4 * 1024 * 1024, "write_buffer_size": 4 * 1024 * 1024,
"writepercent": 35, "writepercent": 35,
"format_version": lambda: random.randint(2, 3),
} }
_TEST_DIR_ENV_VAR = 'TEST_TMPDIR' _TEST_DIR_ENV_VAR = 'TEST_TMPDIR'

12
tools/db_stress.cc
Unescape View File

@ -219,10 +219,14 @@ DEFINE_int32(level0_stop_writes_trigger,
rocksdb::Options().level0_stop_writes_trigger, rocksdb::Options().level0_stop_writes_trigger,
"Number of files in level-0 that will trigger put stop."); "Number of files in level-0 that will trigger put stop.");
DEFINE_int32(block_size, DEFINE_int32(block_size, rocksdb::BlockBasedTableOptions().block_size,
static_cast<int32_t>(rocksdb::BlockBasedTableOptions().block_size),
"Number of bytes in a block."); "Number of bytes in a block.");
DEFINE_int32(
format_version,
static_cast<int32_t>(rocksdb::BlockBasedTableOptions().format_version),
"Format version of SST files.");
DEFINE_int32(max_background_compactions, DEFINE_int32(max_background_compactions,
rocksdb::Options().max_background_compactions, rocksdb::Options().max_background_compactions,
"The maximum number of concurrent background compactions " "The maximum number of concurrent background compactions "
@ -2480,6 +2484,7 @@ class StressTest {
void PrintEnv() const { void PrintEnv() const {
fprintf(stdout, "RocksDB version : %d.%d\n", kMajorVersion, fprintf(stdout, "RocksDB version : %d.%d\n", kMajorVersion,
kMinorVersion); kMinorVersion);
fprintf(stdout, "Format version : %d\n", FLAGS_format_version);
fprintf(stdout, "TransactionDB : %s\n", fprintf(stdout, "TransactionDB : %s\n",
FLAGS_use_txn ? "true" : "false"); FLAGS_use_txn ? "true" : "false");
fprintf(stdout, "Column families : %d\n", FLAGS_column_families); fprintf(stdout, "Column families : %d\n", FLAGS_column_families);
@ -2563,7 +2568,8 @@ class StressTest {
block_based_options.block_cache_compressed = compressed_cache_; block_based_options.block_cache_compressed = compressed_cache_;
block_based_options.checksum = FLAGS_checksum_type_e; block_based_options.checksum = FLAGS_checksum_type_e;
block_based_options.block_size = FLAGS_block_size; block_based_options.block_size = FLAGS_block_size;
block_based_options.format_version = 2; block_based_options.format_version =
static_cast<uint32_t>(FLAGS_format_version);
block_based_options.filter_policy = filter_policy_; block_based_options.filter_policy = filter_policy_;
options_.table_factory.reset( options_.table_factory.reset(
NewBlockBasedTableFactory(block_based_options)); NewBlockBasedTableFactory(block_based_options));

12
util/testutil.cc
Unescape View File

@ -19,6 +19,9 @@
namespace rocksdb { namespace rocksdb {
namespace test { namespace test {
const uint32_t kDefaultFormatVersion = BlockBasedTableOptions().format_version;
const uint32_t kLatestFormatVersion = 3u;
Slice RandomString(Random* rnd, int len, std::string* dst) { Slice RandomString(Random* rnd, int len, std::string* dst) {
dst->resize(len); dst->resize(len);
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
@ -116,14 +119,9 @@ class Uint64ComparatorImpl : public Comparator {
}; };
} // namespace } // namespace
static port::OnceType once;
static const Comparator* uint64comp;
static void InitModule() { uint64comp = new Uint64ComparatorImpl; }
const Comparator* Uint64Comparator() { const Comparator* Uint64Comparator() {
port::InitOnce(&once, InitModule); static Uint64ComparatorImpl uint64comp;
return uint64comp; return &uint64comp;
} }
WritableFileWriter* GetWritableFileWriter(WritableFile* wf) { WritableFileWriter* GetWritableFileWriter(WritableFile* wf) {

3
util/testutil.h
Unescape View File

@ -32,6 +32,9 @@ class SequentialFileReader;
namespace test { namespace test {
extern const uint32_t kDefaultFormatVersion;
extern const uint32_t kLatestFormatVersion;
// Store in *dst a random string of length "len" and return a Slice that // Store in *dst a random string of length "len" and return a Slice that
// references the generated data. // references the generated data.
extern Slice RandomString(Random* rnd, int len, std::string* dst); extern Slice RandomString(Random* rnd, int len, std::string* dst);

Write Preview
Loading…
Cancel
Save