NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update blob_db_test

Browse Source 
Summary:
Re-enable blob_db_test with some update:
* Commented out delay at the end of GC tests. Will update the logic later with sync point to properly trigger GC.
* Added some helper functions.

Also update make files to include blob_dump tool.
Closes https://github.com/facebook/rocksdb/pull/2375

Differential Revision: D5133793

Pulled By: yiwu-arbug

fbshipit-source-id: 95470b26d0c1f9592ba4b7637e027fdd263f425c
main
Yi Wu 7 years ago committed by Facebook Github Bot
parent cbc821c25b
commit 345878a7fb
5 changed files with 108 additions and 172 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      CMakeLists.txt
  2. 3
      Makefile
  3. 5
      TARGETS
  4. 1
      src.mk
  5. 269
      utilities/blob_db/blob_db_test.cc

2
CMakeLists.txt
Unescape View File

@ -475,6 +475,7 @@ set(SOURCES
utilities/blob_db/blob_db.cc
utilities/blob_db/blob_db_impl.cc
utilities/blob_db/blob_db_options_impl.cc
utilities/blob_db/blob_dump_tool.cc
utilities/blob_db/blob_file.cc
utilities/blob_db/blob_log_reader.cc
utilities/blob_db/blob_log_writer.cc
@ -696,6 +697,7 @@ set(TESTS
util/thread_list_test.cc
util/thread_local_test.cc
utilities/backupable/backupable_db_test.cc
utilities/blob_db/blob_db_test.cc
utilities/checkpoint/checkpoint_test.cc
utilities/column_aware_encoding_test.cc
utilities/date_tiered/date_tiered_test.cc

3
Makefile
Unescape View File

@ -1377,6 +1377,9 @@ lua_test: utilities/lua/rocks_lua_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTH
range_del_aggregator_test: db/range_del_aggregator_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
blob_db_test: utilities/blob_db/blob_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
#-------------------------------------------------
# make install related stuff
INSTALL_PATH ?= /usr/local

5
TARGETS
Unescape View File

@ -200,6 +200,7 @@ cpp_library(
"utilities/blob_db/blob_db.cc",
"utilities/blob_db/blob_db_impl.cc",
"utilities/blob_db/blob_db_options_impl.cc",
"utilities/blob_db/blob_dump_tool.cc",
"utilities/blob_db/blob_file.cc",
"utilities/blob_db/blob_log_reader.cc",
"utilities/blob_db/blob_log_writer.cc",
@ -456,8 +457,8 @@ ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
['version_edit_test', 'db/version_edit_test.cc', 'serial'],
['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial']]
['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial'],
['blob_db_test', 'utilities/blob_db/blob_db_test.cc', 'serial']]
# Generate a test rule for each entry in ROCKS_TESTS
for test_cfg in ROCKS_TESTS:

1
src.mk
Unescape View File

@ -320,6 +320,7 @@ MAIN_SOURCES = \
util/thread_list_test.cc \
util/thread_local_test.cc \
utilities/backupable/backupable_db_test.cc \
utilities/blob_db/blob_db_test.cc \
utilities/checkpoint/checkpoint_test.cc \
utilities/column_aware_encoding_exp.cc \
utilities/column_aware_encoding_test.cc \

269
utilities/blob_db/blob_db_test.cc
Unescape View File

@ -7,10 +7,15 @@
#include "utilities/blob_db/blob_db.h"
#include <cstdlib>
#include <map>
#include <memory>
#include <string>
#include "db/db_test_util.h"
#include "port/port.h"
#include "util/random.h"
#include "util/string_util.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "utilities/blob_db/blob_db_impl.h"
#include "utilities/blob_db/blob_db_options_impl.h"
namespace rocksdb {
@ -33,9 +38,11 @@ void gen_random(char *s, const int len) {
class BlobDBTest : public testing::Test {
public:
const int kMaxBlobSize = 1 << 14;
BlobDBTest() : blobdb_(nullptr) {
dbname_ = test::TmpDir() + "/blob_db_test";
// Reopen1(BlobDBOptionsImpl());
// Reopen(BlobDBOptionsImpl());
}
~BlobDBTest() {
@ -45,8 +52,8 @@ class BlobDBTest : public testing::Test {
}
}
void Reopen1(const BlobDBOptionsImpl &bdboptions,
const Options &options = Options()) {
void Reopen(const BlobDBOptionsImpl &bdboptions,
const Options &options = Options()) {
if (blobdb_) {
delete blobdb_;
blobdb_ = nullptr;
@ -61,40 +68,59 @@ class BlobDBTest : public testing::Test {
myoptions.create_if_missing = true;
EXPECT_TRUE(
BlobDB::Open(myoptions, bblobdb_options, dbname_, &blobdb_).ok());
ASSERT_NE(nullptr, blobdb_);
}
void insert_blobs() {
WriteOptions wo;
ReadOptions ro;
std::string value;
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
Random rnd(301);
for (size_t i = 0; i < 100000; i++) {
int len = rnd.Next() % 16384;
if (!len) continue;
void PutRandomWithTTL(const std::string &key, int32_t ttl, Random *rnd,
std::map<std::string, std::string> *data = nullptr) {
int len = rnd->Next() % kMaxBlobSize + 1;
std::string value = test::RandomHumanReadableString(rnd, len);
ColumnFamilyHandle *cfh = blobdb_->DefaultColumnFamily();
ASSERT_OK(blobdb_->PutWithTTL(WriteOptions(), cfh, Slice(key), Slice(value),
ttl));
if (data != nullptr) {
(*data)[key] = value;
}
}
char *val = new char[len + 1];
gen_random(val, len);
void PutRandom(const std::string &key, Random *rnd,
std::map<std::string, std::string> *data = nullptr) {
PutRandomWithTTL(key, -1, rnd, data);
}
std::string key("key");
key += std::to_string(i % 500);
void Delete(const std::string &key) {
ColumnFamilyHandle *cfh = blobdb_->DefaultColumnFamily();
ASSERT_OK(blobdb_->Delete(WriteOptions(), cfh, key));
}
Slice keyslice(key);
Slice valslice(val, len + 1);
// Verify blob db contain expected data and nothing more.
// TODO(yiwu): Verify blob files are consistent with data in LSM.
void VerifyDB(const std::map<std::string, std::string> &data) {
Iterator *iter = blobdb_->NewIterator(ReadOptions());
iter->SeekToFirst();
for (auto &p : data) {
ASSERT_TRUE(iter->Valid());
ASSERT_EQ(p.first, iter->key().ToString());
ASSERT_EQ(p.second, iter->value().ToString());
iter->Next();
}
ASSERT_FALSE(iter->Valid());
ASSERT_OK(iter->status());
delete iter;
}
int ttl = rnd.Next() % 86400;
void InsertBlobs() {
WriteOptions wo;
std::string value;
ASSERT_OK(blobdb_->PutWithTTL(wo, dcfh, keyslice, valslice, ttl));
delete[] val;
Random rnd(301);
for (size_t i = 0; i < 100000; i++) {
int32_t ttl = rnd.Next() % 86400;
PutRandomWithTTL("key" + ToString(i % 500), ttl, &rnd, nullptr);
}
for (size_t i = 0; i < 10; i++) {
std::string key("key");
key += std::to_string(i % 500);
Slice keyslice(key);
blobdb_->Delete(wo, dcfh, keyslice);
Delete("key" + ToString(i % 500));
}
}
@ -104,162 +130,60 @@ class BlobDBTest : public testing::Test {
TEST_F(BlobDBTest, DeleteComplex) {
BlobDBOptionsImpl bdboptions;
bdboptions.partial_expiration_pct = 75;
bdboptions.gc_check_period_millisecs = 20 * 1000;
bdboptions.blob_file_size = 219 * 1024;
Reopen1(bdboptions);
WriteOptions wo;
ReadOptions ro;
std::string value;
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
Reopen(bdboptions);
Random rnd(301);
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
if (!len) continue;
char *val = new char[len + 1];
gen_random(val, len);
std::string key("key");
key += std::to_string(i);
Slice keyslice(key);
Slice valslice(val, len + 1);
ASSERT_OK(blobdb_->Put(wo, dcfh, keyslice, valslice));
delete[] val;
PutRandom("key" + ToString(i), &rnd, nullptr);
}
for (size_t i = 0; i < 99; i++) {
std::string key("key");
key += std::to_string(i);
Slice keyslice(key);
blobdb_->Delete(wo, dcfh, keyslice);
for (size_t i = 0; i < 100; i++) {
Delete("key" + ToString(i));
}
Env::Default()->SleepForMicroseconds(60 * 1000 * 1000);
// DB should be empty.
VerifyDB({});
}
TEST_F(BlobDBTest, OverrideTest) {
BlobDBOptionsImpl bdboptions;
bdboptions.ttl_range_secs = 30;
bdboptions.gc_file_pct = 100;
bdboptions.gc_check_period_millisecs = 20 * 1000;
bdboptions.num_concurrent_simple_blobs = 2;
bdboptions.blob_file_size = 876 * 1024 * 10;
Options options;
options.write_buffer_size = 256 * 1024;
options.info_log_level = INFO_LEVEL;
Reopen1(bdboptions, options);
WriteOptions wo;
ReadOptions ro;
std::string value;
Reopen(bdboptions, options);
Random rnd(301);
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
std::map<std::string, std::string> data;
for (int i = 0; i < 10000; i++) {
int len = rnd.Next() % 16384;
if (!len) continue;
char *val = new char[len + 1];
gen_random(val, len);
std::string key("key");
char x[10];
std::sprintf(x, "%04d", i);
key += std::string(x);
Slice keyslice(key);
Slice valslice(val, len + 1);
ASSERT_OK(blobdb_->Put(wo, dcfh, keyslice, valslice));
delete[] val;
PutRandom("key" + ToString(i), &rnd, nullptr);
}
// override all the keys
for (int i = 0; i < 10000; i++) {
int len = rnd.Next() % 16384;
if (!len) continue;
char *val = new char[len + 1];
gen_random(val, len);
std::string key("key");
char x[10];
std::sprintf(x, "%04d", i);
key += std::string(x);
Slice keyslice(key);
Slice valslice(val, len + 1);
ASSERT_OK(blobdb_->Put(wo, dcfh, keyslice, valslice));
delete[] val;
PutRandom("key" + ToString(i), &rnd, &data);
}
blobdb_->Flush(FlushOptions());
#if 1
blobdb_->GetBaseDB()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
reinterpret_cast<DBImpl *>(blobdb_->GetBaseDB())->TEST_WaitForFlushMemTable();
reinterpret_cast<DBImpl *>(blobdb_->GetBaseDB())->TEST_WaitForCompact();
#endif
Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
VerifyDB(data);
}
TEST_F(BlobDBTest, DeleteTest) {
BlobDBOptionsImpl bdboptions;
bdboptions.ttl_range_secs = 30;
bdboptions.gc_file_pct = 100;
bdboptions.partial_expiration_pct = 18;
bdboptions.gc_check_period_millisecs = 20 * 1000;
bdboptions.num_concurrent_simple_blobs = 1;
bdboptions.blob_file_size = 876 * 1024;
Reopen1(bdboptions);
WriteOptions wo;
ReadOptions ro;
std::string value;
Reopen(bdboptions);
Random rnd(301);
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
std::map<std::string, std::string> data;
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
if (!len) continue;
char *val = new char[len + 1];
gen_random(val, len);
std::string key("key");
key += std::to_string(i);
Slice keyslice(key);
Slice valslice(val, len + 1);
ASSERT_OK(blobdb_->Put(wo, dcfh, keyslice, valslice));
delete[] val;
PutRandom("key" + ToString(i), &rnd, &data);
}
for (size_t i = 0; i < 100; i += 5) {
std::string key("key");
key += std::to_string(i);
Slice keyslice(key);
blobdb_->Delete(wo, dcfh, keyslice);
Delete("key" + ToString(i));
data.erase("key" + ToString(i));
}
Env::Default()->SleepForMicroseconds(60 * 1000 * 1000);
VerifyDB(data);
}
TEST_F(BlobDBTest, GCTestWithWrite) {
@ -269,7 +193,7 @@ TEST_F(BlobDBTest, GCTestWithWrite) {
bdboptions.gc_check_period_millisecs = 20 * 1000;
bdboptions.default_ttl_extractor = true;
Reopen1(bdboptions);
Reopen(bdboptions);
WriteOptions wo;
ReadOptions ro;
@ -277,11 +201,11 @@ TEST_F(BlobDBTest, GCTestWithWrite) {
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
WriteBatch WB;
WriteBatch batch;
Random rnd(301);
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
int len = rnd.Next() % kMaxBlobSize;
if (!len) continue;
int ttl = 30;
@ -297,13 +221,14 @@ TEST_F(BlobDBTest, GCTestWithWrite) {
Slice keyslice(key);
Slice valslice(val, len + BlobDB::kTTLSuffixLength);
WB.Put(dcfh, keyslice, valslice);
batch.Put(dcfh, keyslice, valslice);
delete[] val;
}
ASSERT_OK(blobdb_->Write(wo, &WB));
ASSERT_OK(blobdb_->Write(wo, &batch));
Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
// TODO(yiwu): Use sync point to properly trigger GC and check result.
// Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
}
void cb_evict(const ColumnFamilyHandle *cfh, const Slice &key,
@ -348,7 +273,7 @@ TEST_F(BlobDBTest, GetWithCompression) {
bdboptions.gc_evict_cb_fn = &cb_evict;
bdboptions.compression = CompressionType::kLZ4Compression;
Reopen1(bdboptions);
Reopen(bdboptions);
WriteOptions wo;
ReadOptions ro;
@ -360,7 +285,7 @@ TEST_F(BlobDBTest, GetWithCompression) {
std::string orig(LONG_STRING);
for (size_t i = 0; i < 10000; i++) {
int len = orig.length();
size_t len = orig.length();
int ttl = 3000 * (rnd.Next() % 10);
char *val = new char[len + BlobDB::kTTLSuffixLength];
@ -388,7 +313,8 @@ TEST_F(BlobDBTest, GetWithCompression) {
ASSERT_TRUE(orig == val);
}
Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
// TODO(yiwu): Use sync point to properly trigger GC and check result.
// Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
}
TEST_F(BlobDBTest, GCTestWithPutAndCompression) {
@ -400,7 +326,7 @@ TEST_F(BlobDBTest, GCTestWithPutAndCompression) {
bdboptions.gc_evict_cb_fn = &cb_evict;
bdboptions.compression = CompressionType::kLZ4Compression;
Reopen1(bdboptions);
Reopen(bdboptions);
WriteOptions wo;
ReadOptions ro;
@ -410,7 +336,7 @@ TEST_F(BlobDBTest, GCTestWithPutAndCompression) {
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
int len = rnd.Next() % kMaxBlobSize;
if (!len) continue;
int ttl = 30;
@ -430,7 +356,8 @@ TEST_F(BlobDBTest, GCTestWithPutAndCompression) {
delete[] val;
}
Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
// TODO(yiwu): Use sync point to properly trigger GC and check result.
// Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
}
TEST_F(BlobDBTest, GCTestWithPut) {
@ -441,7 +368,7 @@ TEST_F(BlobDBTest, GCTestWithPut) {
bdboptions.default_ttl_extractor = true;
bdboptions.gc_evict_cb_fn = &cb_evict;
Reopen1(bdboptions);
Reopen(bdboptions);
WriteOptions wo;
ReadOptions ro;
@ -451,7 +378,7 @@ TEST_F(BlobDBTest, GCTestWithPut) {
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
int len = rnd.Next() % kMaxBlobSize;
if (!len) continue;
int ttl = 30;
@ -471,7 +398,8 @@ TEST_F(BlobDBTest, GCTestWithPut) {
delete[] val;
}
Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
// TODO(yiwu): Use sync point to properly trigger GC and check result.
// Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
}
TEST_F(BlobDBTest, GCTest) {
@ -479,7 +407,7 @@ TEST_F(BlobDBTest, GCTest) {
bdboptions.ttl_range_secs = 30;
bdboptions.gc_file_pct = 100;
Reopen1(bdboptions);
Reopen(bdboptions);
WriteOptions wo;
ReadOptions ro;
@ -489,7 +417,7 @@ TEST_F(BlobDBTest, GCTest) {
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
int len = rnd.Next() % kMaxBlobSize;
if (!len) continue;
char *val = new char[len + 1];
@ -507,18 +435,19 @@ TEST_F(BlobDBTest, GCTest) {
delete[] val;
}
Env::Default()->SleepForMicroseconds(240 * 1000 * 1000);
// TODO(yiwu): Use sync point to properly trigger GC and check result.
// Env::Default()->SleepForMicroseconds(240 * 1000 * 1000);
}
TEST_F(BlobDBTest, DISABLED_MultipleWriters) {
BlobDBOptionsImpl bdboptions;
Reopen1(bdboptions);
Reopen(bdboptions);
ASSERT_TRUE(blobdb_ != nullptr);
std::vector<std::thread> workers;
for (size_t ii = 0; ii < 10; ii++)
workers.push_back(std::thread(&BlobDBTest::insert_blobs, this));
workers.push_back(std::thread(&BlobDBTest::InsertBlobs, this));
for (std::thread &t : workers) {
if (t.joinable()) {
@ -534,9 +463,10 @@ TEST_F(BlobDBTest, DISABLED_MultipleWriters) {
// ASSERT_EQ("v2", value);
}
#if 0
TEST_F(BlobDBTest, Large) {
ASSERT_TRUE(blobdb_ != nullptr);
BlobDBOptionsImpl bdboptions;
Options options;
Reopen(bdboptions, options);
WriteOptions wo;
ReadOptions ro;
@ -559,7 +489,6 @@ TEST_F(BlobDBTest, Large) {
ASSERT_OK(blobdb_->Get(ro, dcfh, "barfoo", &value));
ASSERT_EQ(value3, value);
}
#endif
} // namespace blob_db
} // namespace rocksdb

Write Preview
Loading…
Cancel
Save