NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use FLAGS_env for certain operations in db_bench (#5943)

Browse Source 
Summary:
Since we already parse env_uri from command line and creates custom Env
accordingly, we should invoke the methods of such Envs instead of using
Env::Default().

Test Plan (on devserver):
```
$make db_bench db_stress
$./db_bench -benchmarks=fillseq
./db_stress
```
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5943

Differential Revision: D18018550

Pulled By: riversand963

fbshipit-source-id: 03b61329aaae0dfd914a0b902cc677f570f102e3
main
Yanqin Jin 5 years ago committed by Facebook Github Bot
parent 925250f42f
commit c0abc6bbc1
2 changed files with 38 additions and 13 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 16
      tools/db_bench_tool.cc
  2. 35
      tools/db_stress_tool.cc

16
tools/db_bench_tool.cc
Unescape View File

@ -889,6 +889,9 @@ DEFINE_string(env_uri, "", "URI for registry Env lookup. Mutually exclusive"
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
DEFINE_string(hdfs, "", "Name of hdfs environment. Mutually exclusive with" DEFINE_string(hdfs, "", "Name of hdfs environment. Mutually exclusive with"
" --env_uri."); " --env_uri.");
static std::shared_ptr<rocksdb::Env> env_guard;
static rocksdb::Env* FLAGS_env = rocksdb::Env::Default(); static rocksdb::Env* FLAGS_env = rocksdb::Env::Default();
DEFINE_int64(stats_interval, 0, "Stats are reported every N operations when " DEFINE_int64(stats_interval, 0, "Stats are reported every N operations when "
@ -1738,7 +1741,7 @@ class Stats {
"ElapsedTime", "Stage", "State", "OperationProperties"); "ElapsedTime", "Stage", "State", "OperationProperties");
int64_t current_time = 0; int64_t current_time = 0;
Env::Default()->GetCurrentTime(&current_time); FLAGS_env->GetCurrentTime(&current_time);
for (auto ts : thread_list) { for (auto ts : thread_list) {
fprintf(stderr, "%18" PRIu64 " %10s %12s %20s %13s %45s %12s", fprintf(stderr, "%18" PRIu64 " %10s %12s %20s %13s %45s %12s",
ts.thread_id, ts.thread_id,
@ -2483,7 +2486,7 @@ class Benchmark {
"at the same time"); "at the same time");
exit(1); exit(1);
} }
FLAGS_env = new ReportFileOpEnv(rocksdb::Env::Default()); FLAGS_env = new ReportFileOpEnv(FLAGS_env);
} }
if (FLAGS_prefix_size > FLAGS_key_size) { if (FLAGS_prefix_size > FLAGS_key_size) {
@ -2500,6 +2503,7 @@ class Benchmark {
} }
if (!FLAGS_use_existing_db) { if (!FLAGS_use_existing_db) {
Options options; Options options;
options.env = FLAGS_env;
if (!FLAGS_wal_dir.empty()) { if (!FLAGS_wal_dir.empty()) {
options.wal_dir = FLAGS_wal_dir; options.wal_dir = FLAGS_wal_dir;
} }
@ -3377,8 +3381,9 @@ class Benchmark {
DBOptions db_opts; DBOptions db_opts;
std::vector<ColumnFamilyDescriptor> cf_descs; std::vector<ColumnFamilyDescriptor> cf_descs;
if (FLAGS_options_file != "") { if (FLAGS_options_file != "") {
auto s = LoadOptionsFromFile(FLAGS_options_file, Env::Default(), &db_opts, auto s = LoadOptionsFromFile(FLAGS_options_file, FLAGS_env, &db_opts,
&cf_descs); &cf_descs);
db_opts.env = FLAGS_env;
if (s.ok()) { if (s.ok()) {
*opts = Options(db_opts, cf_descs[0].options); *opts = Options(db_opts, cf_descs[0].options);
return true; return true;
@ -3399,6 +3404,7 @@ class Benchmark {
assert(db_.db == nullptr); assert(db_.db == nullptr);
options.env = FLAGS_env;
options.max_open_files = FLAGS_open_files; options.max_open_files = FLAGS_open_files;
if (FLAGS_cost_write_buffer_to_cache || FLAGS_db_write_buffer_size != 0) { if (FLAGS_cost_write_buffer_to_cache || FLAGS_db_write_buffer_size != 0) {
options.write_buffer_manager.reset( options.write_buffer_manager.reset(
@ -6596,7 +6602,7 @@ int db_bench_tool(int argc, char** argv) {
fprintf(stderr, "Cannot provide both --hdfs and --env_uri.\n"); fprintf(stderr, "Cannot provide both --hdfs and --env_uri.\n");
exit(1); exit(1);
} else if (!FLAGS_env_uri.empty()) { } else if (!FLAGS_env_uri.empty()) {
Status s = Env::LoadEnv(FLAGS_env_uri, &FLAGS_env); Status s = Env::LoadEnv(FLAGS_env_uri, &FLAGS_env, &env_guard);
if (FLAGS_env == nullptr) { if (FLAGS_env == nullptr) {
fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str()); fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str());
exit(1); exit(1);
@ -6641,7 +6647,7 @@ int db_bench_tool(int argc, char** argv) {
// Choose a location for the test database if none given with --db=<path> // Choose a location for the test database if none given with --db=<path>
if (FLAGS_db.empty()) { if (FLAGS_db.empty()) {
std::string default_db_path; std::string default_db_path;
rocksdb::Env::Default()->GetTestDirectory(&default_db_path); FLAGS_env->GetTestDirectory(&default_db_path);
default_db_path += "/dbbench"; default_db_path += "/dbbench";
FLAGS_db = default_db_path; FLAGS_db = default_db_path;
} }

35
tools/db_stress_tool.cc
Unescape View File

@ -617,6 +617,11 @@ DEFINE_string(checksum_type, "kCRC32c", "Algorithm to use to checksum blocks");
static enum rocksdb::ChecksumType FLAGS_checksum_type_e = rocksdb::kCRC32c; static enum rocksdb::ChecksumType FLAGS_checksum_type_e = rocksdb::kCRC32c;
DEFINE_string(hdfs, "", "Name of hdfs environment"); DEFINE_string(hdfs, "", "Name of hdfs environment");
DEFINE_string(env_uri, "",
"URI for env lookup. Mutually exclusive with --hdfs");
static std::shared_ptr<rocksdb::Env> env_guard;
// posix or hdfs environment // posix or hdfs environment
static rocksdb::Env* FLAGS_env = rocksdb::Env::Default(); static rocksdb::Env* FLAGS_env = rocksdb::Env::Default();
@ -2721,7 +2726,9 @@ class StressTest {
assert(rand_column_families.size() == rand_keys.size()); assert(rand_column_families.size() == rand_keys.size());
std::string checkpoint_dir = std::string checkpoint_dir =
FLAGS_db + "/.checkpoint" + ToString(thread->tid); FLAGS_db + "/.checkpoint" + ToString(thread->tid);
DestroyDB(checkpoint_dir, Options()); Options tmp_opts(options_);
tmp_opts.listeners.clear();
DestroyDB(checkpoint_dir, tmp_opts);
Checkpoint* checkpoint = nullptr; Checkpoint* checkpoint = nullptr;
Status s = Checkpoint::Create(db_, &checkpoint); Status s = Checkpoint::Create(db_, &checkpoint);
if (s.ok()) { if (s.ok()) {
@ -2777,7 +2784,7 @@ class StressTest {
delete checkpoint_db; delete checkpoint_db;
checkpoint_db = nullptr; checkpoint_db = nullptr;
} }
DestroyDB(checkpoint_dir, Options()); DestroyDB(checkpoint_dir, tmp_opts);
if (!s.ok()) { if (!s.ok()) {
fprintf(stderr, "A checkpoint operation failed with: %s\n", fprintf(stderr, "A checkpoint operation failed with: %s\n",
s.ToString().c_str()); s.ToString().c_str());
@ -2984,8 +2991,9 @@ class StressTest {
#else #else
DBOptions db_options; DBOptions db_options;
std::vector<ColumnFamilyDescriptor> cf_descriptors; std::vector<ColumnFamilyDescriptor> cf_descriptors;
Status s = LoadOptionsFromFile(FLAGS_options_file, Env::Default(), Status s = LoadOptionsFromFile(FLAGS_options_file, FLAGS_env, &db_options,
&db_options, &cf_descriptors); &cf_descriptors);
db_options.env = FLAGS_env;
if (!s.ok()) { if (!s.ok()) {
fprintf(stderr, "Unable to load options file %s --- %s\n", fprintf(stderr, "Unable to load options file %s --- %s\n",
FLAGS_options_file.c_str(), s.ToString().c_str()); FLAGS_options_file.c_str(), s.ToString().c_str());
@ -3169,6 +3177,7 @@ class StressTest {
secondaries_.resize(FLAGS_threads); secondaries_.resize(FLAGS_threads);
std::fill(secondaries_.begin(), secondaries_.end(), nullptr); std::fill(secondaries_.begin(), secondaries_.end(), nullptr);
Options tmp_opts; Options tmp_opts;
tmp_opts.env = options_.env;
tmp_opts.max_open_files = FLAGS_open_files; tmp_opts.max_open_files = FLAGS_open_files;
for (size_t i = 0; i != static_cast<size_t>(FLAGS_threads); ++i) { for (size_t i = 0; i != static_cast<size_t>(FLAGS_threads); ++i) {
const std::string secondary_path = const std::string secondary_path =
@ -3694,7 +3703,7 @@ class NonBatchedOpsStressTest : public StressTest {
s = FLAGS_env->DeleteFile(sst_filename); s = FLAGS_env->DeleteFile(sst_filename);
} }
SstFileWriter sst_file_writer(EnvOptions(), options_); SstFileWriter sst_file_writer(EnvOptions(options_), options_);
if (s.ok()) { if (s.ok()) {
s = sst_file_writer.Open(sst_filename); s = sst_file_writer.Open(sst_filename);
} }
@ -4324,7 +4333,7 @@ class CfConsistencyStressTest : public StressTest {
const std::vector<int64_t>& /* rand_keys */) { const std::vector<int64_t>& /* rand_keys */) {
std::string checkpoint_dir = std::string checkpoint_dir =
FLAGS_db + "/.checkpoint" + ToString(thread->tid); FLAGS_db + "/.checkpoint" + ToString(thread->tid);
DestroyDB(checkpoint_dir, Options()); DestroyDB(checkpoint_dir, options_);
Checkpoint* checkpoint = nullptr; Checkpoint* checkpoint = nullptr;
Status s = Checkpoint::Create(db_, &checkpoint); Status s = Checkpoint::Create(db_, &checkpoint);
if (s.ok()) { if (s.ok()) {
@ -4358,7 +4367,7 @@ class CfConsistencyStressTest : public StressTest {
delete checkpoint_db; delete checkpoint_db;
checkpoint_db = nullptr; checkpoint_db = nullptr;
} }
DestroyDB(checkpoint_dir, Options()); DestroyDB(checkpoint_dir, options_);
if (!s.ok()) { if (!s.ok()) {
fprintf(stderr, "A checkpoint operation failed with: %s\n", fprintf(stderr, "A checkpoint operation failed with: %s\n",
s.ToString().c_str()); s.ToString().c_str());
@ -4546,7 +4555,17 @@ int db_stress_tool(int argc, char** argv) {
StringToCompressionType(FLAGS_compression_type.c_str()); StringToCompressionType(FLAGS_compression_type.c_str());
FLAGS_checksum_type_e = StringToChecksumType(FLAGS_checksum_type.c_str()); FLAGS_checksum_type_e = StringToChecksumType(FLAGS_checksum_type.c_str());
if (!FLAGS_hdfs.empty()) { if (!FLAGS_hdfs.empty()) {
if (!FLAGS_env_uri.empty()) {
fprintf(stderr, "Cannot specify both --hdfs and --env_uri.\n");
exit(1);
}
FLAGS_env = new rocksdb::HdfsEnv(FLAGS_hdfs); FLAGS_env = new rocksdb::HdfsEnv(FLAGS_hdfs);
} else if (!FLAGS_env_uri.empty()) {
Status s = Env::LoadEnv(FLAGS_env_uri, &FLAGS_env, &env_guard);
if (FLAGS_env == nullptr) {
fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str());
exit(1);
}
} }
FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str()); FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str());
@ -4644,7 +4663,7 @@ int db_stress_tool(int argc, char** argv) {
// Choose a location for the test database if none given with --db=<path> // Choose a location for the test database if none given with --db=<path>
if (FLAGS_db.empty()) { if (FLAGS_db.empty()) {
std::string default_db_path; std::string default_db_path;
rocksdb::Env::Default()->GetTestDirectory(&default_db_path); FLAGS_env->GetTestDirectory(&default_db_path);
default_db_path += "/dbstress"; default_db_path += "/dbstress";
FLAGS_db = default_db_path; FLAGS_db = default_db_path;
} }

Write Preview
Loading…
Cancel
Save