NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Track WAL in MANIFEST: add option track_and_verify_wals_in_manifest (#7275)

Browse Source 
Summary:
This option determines whether WALs will be tracked in MANIFEST and verified on recovery.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7275

Test Plan:
db_options_test
options_test

Reviewed By: pdillinger

Differential Revision: D23181418

Pulled By: cheng-chang

fbshipit-source-id: 5dd1cdc166f3dfc1c93c094df4a2f7734e3b4547
main
Cheng Chang 4 years ago committed by Facebook GitHub Bot
parent 9dd25487cc
commit 12b78e40bd
8 changed files with 51 additions and 0 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 15
      db/db_options_test.cc
  2. 14
      include/rocksdb/options.h
  3. 11
      options/db_options.cc
  4. 1
      options/db_options.h
  5. 2
      options/options_helper.cc
  6. 1
      options/options_settable_test.cc
  7. 6
      options/options_test.cc
  8. 1
      test_util/testutil.cc

15
db/db_options_test.cc
Unescape View File

@ -79,6 +79,21 @@ class DBOptionsTest : public DBTestBase {
#endif // ROCKSDB_LITE
};
TEST_F(DBOptionsTest, ImmutableTrackAndVerifyWalsInManifest) {
Options options;
options.track_and_verify_wals_in_manifest = true;
ImmutableDBOptions db_options(options);
ASSERT_TRUE(db_options.track_and_verify_wals_in_manifest);
Reopen(options);
ASSERT_TRUE(dbfull()->GetDBOptions().track_and_verify_wals_in_manifest);
Status s =
dbfull()->SetDBOptions({{"track_and_verify_wals_in_manifest", "false"}});
ASSERT_FALSE(s.ok());
}
// RocksDB lite don't support dynamic options.
#ifndef ROCKSDB_LITE

14
include/rocksdb/options.h
Unescape View File

@ -390,6 +390,20 @@ struct DBOptions {
// Default: true
bool paranoid_checks = true;
// If true, track WALs in MANIFEST and verify them on recovery.
//
// If a WAL is tracked in MANIFEST but is missing from disk on recovery,
// or the size of the tracked WAL is larger than the WAL's on-disk size,
// an error is reported and recovery is aborted.
//
// If a WAL is not tracked in MANIFEST, then no verification will happen
// during recovery.
//
// Default: false
// FIXME(cheng): This option is part of a work in progress and does not yet
// work
bool track_and_verify_wals_in_manifest = false;
// Use the specified object to interact with the environment,
// e.g. to read/write files, schedule background work, etc. In the near
// future, support for doing storage operations such as read/write files

11
options/db_options.cc
Unescape View File

@ -198,6 +198,11 @@ static std::unordered_map<std::string, OptionTypeInfo>
{offsetof(struct ImmutableDBOptions, paranoid_checks),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"track_and_verify_wals_in_manifest",
{offsetof(struct ImmutableDBOptions,
track_and_verify_wals_in_manifest),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"skip_log_error_on_recovery",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone}},
@ -497,6 +502,8 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
create_missing_column_families(options.create_missing_column_families),
error_if_exists(options.error_if_exists),
paranoid_checks(options.paranoid_checks),
track_and_verify_wals_in_manifest(
options.track_and_verify_wals_in_manifest),
env(options.env),
fs(options.env->GetFileSystem()),
rate_limiter(options.rate_limiter),
@ -579,6 +586,10 @@ void ImmutableDBOptions::Dump(Logger* log) const {
create_if_missing);
ROCKS_LOG_HEADER(log, " Options.paranoid_checks: %d",
paranoid_checks);
ROCKS_LOG_HEADER(log,
" "
"Options.track_and_verify_wals_in_manifest: %d",
track_and_verify_wals_in_manifest);
ROCKS_LOG_HEADER(log, " Options.env: %p",
env);
ROCKS_LOG_HEADER(log, " Options.fs: %s",

1
options/db_options.h
Unescape View File

@ -23,6 +23,7 @@ struct ImmutableDBOptions {
bool create_missing_column_families;
bool error_if_exists;
bool paranoid_checks;
bool track_and_verify_wals_in_manifest;
Env* env;
std::shared_ptr<FileSystem> fs;
std::shared_ptr<RateLimiter> rate_limiter;

2
options/options_helper.cc
Unescape View File

@ -51,6 +51,8 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
immutable_db_options.create_missing_column_families;
options.error_if_exists = immutable_db_options.error_if_exists;
options.paranoid_checks = immutable_db_options.paranoid_checks;
options.track_and_verify_wals_in_manifest =
immutable_db_options.track_and_verify_wals_in_manifest;
options.env = immutable_db_options.env;
options.rate_limiter = immutable_db_options.rate_limiter;
options.sst_file_manager = immutable_db_options.sst_file_manager;

1
options/options_settable_test.cc
Unescape View File

@ -278,6 +278,7 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
"skip_log_error_on_recovery=true;"
"writable_file_max_buffer_size=1048576;"
"paranoid_checks=true;"
"track_and_verify_wals_in_manifest=true;"
"is_fd_close_on_exec=false;"
"bytes_per_sync=4295013613;"
"strict_bytes_per_sync=true;"

6
options/options_test.cc
Unescape View File

@ -109,6 +109,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"create_missing_column_families", "true"},
{"error_if_exists", "false"},
{"paranoid_checks", "true"},
{"track_and_verify_wals_in_manifest", "true"},
{"max_open_files", "32"},
{"max_total_wal_size", "33"},
{"use_fsync", "true"},
@ -263,6 +264,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_db_opt.create_missing_column_families, true);
ASSERT_EQ(new_db_opt.error_if_exists, false);
ASSERT_EQ(new_db_opt.paranoid_checks, true);
ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true);
ASSERT_EQ(new_db_opt.max_open_files, 32);
ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33));
ASSERT_EQ(new_db_opt.use_fsync, true);
@ -774,6 +776,7 @@ TEST_F(OptionsTest, OldInterfaceTest) {
{"create_missing_column_families", "true"},
{"error_if_exists", "false"},
{"paranoid_checks", "true"},
{"track_and_verify_wals_in_manifest", "true"},
{"max_open_files", "32"},
};
ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
@ -781,6 +784,7 @@ TEST_F(OptionsTest, OldInterfaceTest) {
ASSERT_EQ(new_db_opt.create_missing_column_families, true);
ASSERT_EQ(new_db_opt.error_if_exists, false);
ASSERT_EQ(new_db_opt.paranoid_checks, true);
ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true);
ASSERT_EQ(new_db_opt.max_open_files, 32);
db_options_map["unknown_option"] = "1";
ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
@ -1620,6 +1624,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
{"create_missing_column_families", "true"},
{"error_if_exists", "false"},
{"paranoid_checks", "true"},
{"track_and_verify_wals_in_manifest", "true"},
{"max_open_files", "32"},
{"max_total_wal_size", "33"},
{"use_fsync", "true"},
@ -1768,6 +1773,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_db_opt.create_missing_column_families, true);
ASSERT_EQ(new_db_opt.error_if_exists, false);
ASSERT_EQ(new_db_opt.paranoid_checks, true);
ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true);
ASSERT_EQ(new_db_opt.max_open_files, 32);
ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33));
ASSERT_EQ(new_db_opt.use_fsync, true);

1
test_util/testutil.cc
Unescape View File

@ -309,6 +309,7 @@ void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) {
db_opt->error_if_exists = rnd->Uniform(2);
db_opt->is_fd_close_on_exec = rnd->Uniform(2);
db_opt->paranoid_checks = rnd->Uniform(2);
db_opt->track_and_verify_wals_in_manifest = rnd->Uniform(2);
db_opt->skip_log_error_on_recovery = rnd->Uniform(2);
db_opt->skip_stats_update_on_db_open = rnd->Uniform(2);
db_opt->skip_checking_sst_file_sizes_on_db_open = rnd->Uniform(2);

Write Preview
Loading…
Cancel
Save