From 786c3d45ed06d1b82adc98913734e8122730c3d5 Mon Sep 17 00:00:00 2001 From: Levi Tamasi Date: Fri, 20 Dec 2019 10:25:48 -0800 Subject: [PATCH] Support BlobDB in db_stress (#6230) Summary: The patch adds support for BlobDB to `db_stress`. Note that BlobDB currently does not support (amongst other features) Column Families or the `SingleDelete` API, so for now, those should be disabled on the command line when running `db_stress` in BlobDB mode (using `-column_families=1` and `-nooverwritepercent=0`, respectively). Also, some BlobDB features that do not go well with the verification logic in `db_stress` like TTL and FIFO eviction are not supported currently. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6230 Test Plan: ``` ./db_stress -max_key=100000 -use_blob_db -column_families=1 -nooverwritepercent=0 -reopen=1 -blob_db_file_size=1000000 -target_file_size_base=1000000 -blob_db_enable_gc -blob_db_gc_cutoff=0.1 -blob_db_min_blob_size=10 -blob_db_bytes_per_sync=16384 ``` Differential Revision: D19191476 Pulled By: ltamasi fbshipit-source-id: 35840452af8c5e6095249c7fd9a53a119a0985fc --- db_stress_tool/db_stress_common.h | 10 ++++++ db_stress_tool/db_stress_gflags.cc | 26 +++++++++++++++ db_stress_tool/db_stress_test_base.cc | 47 +++++++++++++++++++++++---- 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index bb3db69a0..e13c874aa 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -65,6 +65,7 @@ #include "util/mutexlock.h" #include "util/random.h" #include "util/string_util.h" +#include "utilities/blob_db/blob_db.h" // SyncPoint is not supported in Released Windows Mode. #if !(defined NDEBUG) || !defined(OS_WIN) #include "test_util/sync_point.h" @@ -211,6 +212,15 @@ DECLARE_int32(verify_checksum_one_in); DECLARE_int32(verify_db_one_in); DECLARE_int32(continuous_verification_interval); +#ifndef ROCKSDB_LITE +DECLARE_bool(use_blob_db); +DECLARE_uint64(blob_db_min_blob_size); +DECLARE_uint64(blob_db_bytes_per_sync); +DECLARE_uint64(blob_db_file_size); +DECLARE_bool(blob_db_enable_gc); +DECLARE_double(blob_db_gc_cutoff); +#endif // !ROCKSDB_LITE + const long KB = 1024; const int kRandomValueMaxFactor = 3; const int kValueMaxLen = 100; diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index 722292e52..d776f60b1 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -270,6 +270,32 @@ DEFINE_bool(allow_concurrent_memtable_write, false, DEFINE_bool(enable_write_thread_adaptive_yield, true, "Use a yielding spin loop for brief writer thread waits."); +#ifndef ROCKSDB_LITE +// BlobDB Options +DEFINE_bool(use_blob_db, false, "Use BlobDB."); + +DEFINE_uint64(blob_db_min_blob_size, + rocksdb::blob_db::BlobDBOptions().min_blob_size, + "Smallest blob to store in a file. Blobs smaller than this " + "will be inlined with the key in the LSM tree."); + +DEFINE_uint64(blob_db_bytes_per_sync, + rocksdb::blob_db::BlobDBOptions().bytes_per_sync, + "Sync blob files once per every N bytes written."); + +DEFINE_uint64(blob_db_file_size, + rocksdb::blob_db::BlobDBOptions().blob_file_size, + "Target size of each blob file."); + +DEFINE_bool(blob_db_enable_gc, + rocksdb::blob_db::BlobDBOptions().enable_garbage_collection, + "Enable BlobDB garbage collection."); + +DEFINE_double(blob_db_gc_cutoff, + rocksdb::blob_db::BlobDBOptions().garbage_collection_cutoff, + "Cutoff ratio for BlobDB garbage collection."); +#endif // !ROCKSDB_LITE + static const bool FLAGS_subcompactions_dummy __attribute__((__unused__)) = RegisterFlagValidator(&FLAGS_subcompactions, &ValidateUint32Range); diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index eba61041c..7ce45b46b 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -38,10 +38,20 @@ StressTest::StressTest() FLAGS_env->DeleteFile(FLAGS_db + "/" + files[i]); } } + Options options; // Remove files without preserving manfiest files options.env = FLAGS_env->target(); - Status s = DestroyDB(FLAGS_db, options); + +#ifndef ROCKSDB_LITE + const Status s = !FLAGS_use_blob_db + ? DestroyDB(FLAGS_db, options) + : blob_db::DestroyBlobDB(FLAGS_db, options, + blob_db::BlobDBOptions()); +#else + const Status s = DestroyDB(FLAGS_db, options); +#endif // !ROCKSDB_LITE + if (!s.ok()) { fprintf(stderr, "Cannot destroy original db: %s\n", s.ToString().c_str()); exit(1); @@ -1444,6 +1454,10 @@ void StressTest::PrintEnv() const { fprintf(stdout, "Format version : %d\n", FLAGS_format_version); fprintf(stdout, "TransactionDB : %s\n", FLAGS_use_txn ? "true" : "false"); +#ifndef ROCKSDB_LITE + fprintf(stdout, "BlobDB : %s\n", + FLAGS_use_blob_db ? "true" : "false"); +#endif // !ROCKSDB_LITE fprintf(stdout, "Read only mode : %s\n", FLAGS_read_only ? "true" : "false"); fprintf(stdout, "Atomic flush : %s\n", @@ -1750,12 +1764,31 @@ void StressTest::Open() { new DbStressListener(FLAGS_db, options_.db_paths, cf_descriptors)); options_.create_missing_column_families = true; if (!FLAGS_use_txn) { - if (db_preload_finished_.load() && FLAGS_read_only) { - s = DB::OpenForReadOnly(DBOptions(options_), FLAGS_db, cf_descriptors, - &column_families_, &db_); - } else { - s = DB::Open(DBOptions(options_), FLAGS_db, cf_descriptors, - &column_families_, &db_); +#ifndef ROCKSDB_LITE + if (FLAGS_use_blob_db) { + blob_db::BlobDBOptions blob_db_options; + blob_db_options.min_blob_size = FLAGS_blob_db_min_blob_size; + blob_db_options.bytes_per_sync = FLAGS_blob_db_bytes_per_sync; + blob_db_options.blob_file_size = FLAGS_blob_db_file_size; + blob_db_options.enable_garbage_collection = FLAGS_blob_db_enable_gc; + blob_db_options.garbage_collection_cutoff = FLAGS_blob_db_gc_cutoff; + + blob_db::BlobDB* blob_db = nullptr; + s = blob_db::BlobDB::Open(options_, blob_db_options, FLAGS_db, + cf_descriptors, &column_families_, &blob_db); + if (s.ok()) { + db_ = blob_db; + } + } else +#endif // !ROCKSDB_LITE + { + if (db_preload_finished_.load() && FLAGS_read_only) { + s = DB::OpenForReadOnly(DBOptions(options_), FLAGS_db, cf_descriptors, + &column_families_, &db_); + } else { + s = DB::Open(DBOptions(options_), FLAGS_db, cf_descriptors, + &column_families_, &db_); + } } } else { #ifndef ROCKSDB_LITE