From 6a279037cf5bb23b60f81e24f731376a195cbf11 Mon Sep 17 00:00:00 2001 From: Yanqin Jin Date: Wed, 18 Sep 2019 11:49:41 -0700 Subject: [PATCH] Refactor ObsoleteFilesTest to inherit from DBTestBase (#5820) Summary: Make class ObsoleteFilesTest inherit from DBTestBase. Test plan (on devserver): ``` $COMPILE_WITH_ASAN=1 make obsolete_files_test $./obsolete_files_test ``` Pull Request resolved: https://github.com/facebook/rocksdb/pull/5820 Differential Revision: D17452348 Pulled By: riversand963 fbshipit-source-id: b09f4581a18022ca2bfd79f2836c0bf7083f5f25 --- Makefile | 2 +- db/obsolete_files_test.cc | 168 ++++++++++++++------------------------ 2 files changed, 62 insertions(+), 108 deletions(-) diff --git a/Makefile b/Makefile index 898570f84..b584b8137 100644 --- a/Makefile +++ b/Makefile @@ -1512,7 +1512,7 @@ options_file_test: db/options_file_test.o $(LIBOBJECTS) $(TESTHARNESS) deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) -obsolete_files_test: db/obsolete_files_test.o $(LIBOBJECTS) $(TESTHARNESS) +obsolete_files_test: db/obsolete_files_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) rocksdb_dump: tools/dump/rocksdb_dump.o $(LIBOBJECTS) diff --git a/db/obsolete_files_test.cc b/db/obsolete_files_test.cc index 305d1e3a3..096a50c1a 100644 --- a/db/obsolete_files_test.cc +++ b/db/obsolete_files_test.cc @@ -14,9 +14,11 @@ #include #include #include "db/db_impl/db_impl.h" +#include "db/db_test_util.h" #include "db/version_set.h" #include "db/write_batch_internal.h" #include "file/filename.h" +#include "port/stack_trace.h" #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/transaction_log.h" @@ -32,60 +34,10 @@ using std::flush; namespace rocksdb { -class ObsoleteFilesTest : public testing::Test { +class ObsoleteFilesTest : public DBTestBase { public: - std::string dbname_; - Options options_; - DB* db_; - Env* env_; - int numlevels_; - - ObsoleteFilesTest() { - db_ = nullptr; - env_ = Env::Default(); - // Trigger compaction when the number of level 0 files reaches 2. - options_.level0_file_num_compaction_trigger = 2; - options_.disable_auto_compactions = false; - options_.delete_obsolete_files_period_micros = 0; // always do full purge - options_.enable_thread_tracking = true; - options_.write_buffer_size = 1024*1024*1000; - options_.target_file_size_base = 1024*1024*1000; - options_.max_bytes_for_level_base = 1024*1024*1000; - options_.WAL_ttl_seconds = 300; // Used to test log files - options_.WAL_size_limit_MB = 1024; // Used to test log files - dbname_ = test::PerThreadDBPath("obsolete_files_test"); - options_.wal_dir = dbname_ + "/wal_files"; - - // clean up all the files that might have been there before - std::vector old_files; - env_->GetChildren(dbname_, &old_files); - for (auto file : old_files) { - env_->DeleteFile(dbname_ + "/" + file); - } - env_->GetChildren(options_.wal_dir, &old_files); - for (auto file : old_files) { - env_->DeleteFile(options_.wal_dir + "/" + file); - } - - DestroyDB(dbname_, options_); - numlevels_ = 7; - EXPECT_OK(ReopenDB(true)); - } - - Status ReopenDB(bool create) { - delete db_; - if (create) { - DestroyDB(dbname_, options_); - } - db_ = nullptr; - options_.create_if_missing = create; - return DB::Open(options_, dbname_, &db_); - } - - void CloseDB() { - delete db_; - db_ = nullptr; - } + ObsoleteFilesTest() + : DBTestBase("/obsolete_files_test"), wal_dir_(dbname_ + "/wal_files") {} void AddKeys(int numkeys, int startkey) { WriteOptions options; @@ -98,50 +50,24 @@ class ObsoleteFilesTest : public testing::Test { } } - int numKeysInLevels( - std::vector &metadata, - std::vector *keysperlevel = nullptr) { - - if (keysperlevel != nullptr) { - keysperlevel->resize(numlevels_); - } - - int numKeys = 0; - for (size_t i = 0; i < metadata.size(); i++) { - int startkey = atoi(metadata[i].smallestkey.c_str()); - int endkey = atoi(metadata[i].largestkey.c_str()); - int numkeysinfile = (endkey - startkey + 1); - numKeys += numkeysinfile; - if (keysperlevel != nullptr) { - (*keysperlevel)[(int)metadata[i].level] += numkeysinfile; - } - fprintf(stderr, "level %d name %s smallest %s largest %s\n", - metadata[i].level, metadata[i].name.c_str(), - metadata[i].smallestkey.c_str(), - metadata[i].largestkey.c_str()); - } - return numKeys; - } - void createLevel0Files(int numFiles, int numKeysPerFile) { int startKey = 0; - DBImpl* dbi = reinterpret_cast(db_); for (int i = 0; i < numFiles; i++) { AddKeys(numKeysPerFile, startKey); startKey += numKeysPerFile; - ASSERT_OK(dbi->TEST_FlushMemTable()); - ASSERT_OK(dbi->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_FlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); } } - void CheckFileTypeCounts(std::string& dir, - int required_log, - int required_sst, - int required_manifest) { + void CheckFileTypeCounts(const std::string& dir, int required_log, + int required_sst, int required_manifest) { std::vector filenames; env_->GetChildren(dir, &filenames); - int log_cnt = 0, sst_cnt = 0, manifest_cnt = 0; + int log_cnt = 0; + int sst_cnt = 0; + int manifest_cnt = 0; for (auto file : filenames) { uint64_t number; FileType type; @@ -155,9 +81,31 @@ class ObsoleteFilesTest : public testing::Test { ASSERT_EQ(required_sst, sst_cnt); ASSERT_EQ(required_manifest, manifest_cnt); } + + void ReopenDB() { + Options options = CurrentOptions(); + // Trigger compaction when the number of level 0 files reaches 2. + options.create_if_missing = true; + options.level0_file_num_compaction_trigger = 2; + options.disable_auto_compactions = false; + options.delete_obsolete_files_period_micros = 0; // always do full purge + options.enable_thread_tracking = true; + options.write_buffer_size = 1024 * 1024 * 1000; + options.target_file_size_base = 1024 * 1024 * 1000; + options.max_bytes_for_level_base = 1024 * 1024 * 1000; + options.WAL_ttl_seconds = 300; // Used to test log files + options.WAL_size_limit_MB = 1024; // Used to test log files + options.wal_dir = wal_dir_; + Destroy(options); + Reopen(options); + } + + const std::string wal_dir_; }; TEST_F(ObsoleteFilesTest, RaceForObsoleteFileDeletion) { + ReopenDB(); + SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->LoadDependency({ {"DBImpl::BackgroundCallCompaction:FoundObsoleteFiles", "ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"}, @@ -178,28 +126,26 @@ TEST_F(ObsoleteFilesTest, RaceForObsoleteFileDeletion) { SyncPoint::GetInstance()->EnableProcessing(); createLevel0Files(2, 50000); - CheckFileTypeCounts(options_.wal_dir, 1, 0, 0); + CheckFileTypeCounts(wal_dir_, 1, 0, 0); - DBImpl* dbi = reinterpret_cast(db_); - port::Thread user_thread([&]() { + port::Thread user_thread([this]() { JobContext jobCxt(0); TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"); - dbi->TEST_LockMutex(); - dbi->FindObsoleteFiles(&jobCxt, - true /* force=true */, false /* no_full_scan=false */); - dbi->TEST_UnlockMutex(); + dbfull()->TEST_LockMutex(); + dbfull()->FindObsoleteFiles(&jobCxt, true /* force=true */, + false /* no_full_scan=false */); + dbfull()->TEST_UnlockMutex(); TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:2"); - dbi->PurgeObsoleteFiles(jobCxt); + dbfull()->PurgeObsoleteFiles(jobCxt); jobCxt.Clean(); }); user_thread.join(); - - CloseDB(); - SyncPoint::GetInstance()->DisableProcessing(); } TEST_F(ObsoleteFilesTest, DeleteObsoleteOptionsFile) { + ReopenDB(); + SyncPoint::GetInstance()->DisableProcessing(); std::vector optsfiles_nums; std::vector optsfiles_keep; SyncPoint::GetInstance()->SetCallBack( @@ -213,23 +159,22 @@ TEST_F(ObsoleteFilesTest, DeleteObsoleteOptionsFile) { SyncPoint::GetInstance()->EnableProcessing(); createLevel0Files(2, 50000); - CheckFileTypeCounts(options_.wal_dir, 1, 0, 0); + CheckFileTypeCounts(wal_dir_, 1, 0, 0); - DBImpl* dbi = static_cast(db_); - ASSERT_OK(dbi->DisableFileDeletions()); + ASSERT_OK(dbfull()->DisableFileDeletions()); for (int i = 0; i != 4; ++i) { if (i % 2) { - ASSERT_OK(dbi->SetOptions(dbi->DefaultColumnFamily(), - {{"paranoid_file_checks", "false"}})); + ASSERT_OK(dbfull()->SetOptions(dbfull()->DefaultColumnFamily(), + {{"paranoid_file_checks", "false"}})); } else { - ASSERT_OK(dbi->SetOptions(dbi->DefaultColumnFamily(), - {{"paranoid_file_checks", "true"}})); + ASSERT_OK(dbfull()->SetOptions(dbfull()->DefaultColumnFamily(), + {{"paranoid_file_checks", "true"}})); } } - ASSERT_OK(dbi->EnableFileDeletions(true /* force */)); + ASSERT_OK(dbfull()->EnableFileDeletions(true /* force */)); ASSERT_EQ(optsfiles_nums.size(), optsfiles_keep.size()); - CloseDB(); + Close(); std::vector files; int opts_file_count = 0; @@ -246,13 +191,22 @@ TEST_F(ObsoleteFilesTest, DeleteObsoleteOptionsFile) { } } ASSERT_EQ(2, opts_file_count); - SyncPoint::GetInstance()->DisableProcessing(); } } //namespace rocksdb +#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS +extern "C" { +void RegisterCustomObjects(int argc, char** argv); +} +#else +void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {} +#endif // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS + int main(int argc, char** argv) { + rocksdb::port::InstallStackTraceHandler(); ::testing::InitGoogleTest(&argc, argv); + RegisterCustomObjects(argc, argv); return RUN_ALL_TESTS(); }