Replace dynamic_cast<>

Summary:
Replace dynamic_cast<> so that users can choose to build with RTTI off, so that they can save several bytes per object, and get tiny more memory available.
Some nontrivial changes:
1. Add Comparator::GetRootComparator() to get around the internal comparator hack
2. Add the two experiemental functions to DB
3. Add TableFactory::GetOptionString() to avoid unnecessary casting to get the option string
4. Since 3 is done, move the parsing option functions for table factory to table factory files too, to be symmetric.
Closes https://github.com/facebook/rocksdb/pull/2645

Differential Revision: D5502723

Pulled By: siying

fbshipit-source-id: fd13cec5601cf68a554d87bfcf056f2ffa5fbf7c
main
Siying Dong 8 years ago committed by Facebook Github Bot
parent e85f2c64cb
commit 21696ba502
  1. 1
      HISTORY.md
  2. 12
      Makefile
  3. 5
      cache/clock_cache.cc
  4. 6
      db/convenience.cc
  5. 6
      db/db_impl.h
  6. 3
      db/dbformat.h
  7. 12
      db/experimental.cc
  8. 5
      db/wal_manager.cc
  9. 4
      env/mock_env.cc
  10. 4
      examples/Makefile
  11. 6
      examples/compaction_filter_example.cc
  12. 4
      include/rocksdb/comparator.h
  13. 11
      include/rocksdb/db.h
  14. 6
      include/rocksdb/table.h
  15. 11
      include/rocksdb/utilities/stackable_db.h
  16. 4
      monitoring/histogram.cc
  17. 5
      monitoring/histogram_windowing.cc
  18. 311
      options/options_helper.cc
  19. 112
      options/options_helper.h
  20. 57
      options/options_parser.cc
  21. 10
      options/options_parser.h
  22. 24
      options/options_test.cc
  23. 193
      table/block_based_table_factory.cc
  24. 94
      table/block_based_table_factory.h
  25. 5
      table/cuckoo_table_factory.h
  26. 144
      table/plain_table_factory.cc
  27. 32
      table/plain_table_factory.h
  28. 6
      tools/db_bench_tool.cc
  29. 7
      tools/ldb_cmd.cc
  30. 12
      tools/sst_dump_tool.cc
  31. 19
      util/cast_util.h
  32. 26
      utilities/blob_db/blob_db_impl.cc
  33. 4
      utilities/column_aware_encoding_util.cc
  34. 24
      utilities/options/options_util_test.cc
  35. 12
      utilities/transactions/optimistic_transaction_impl.cc
  36. 11
      utilities/transactions/transaction_db_impl.cc
  37. 11
      utilities/transactions/transaction_impl.cc
  38. 6
      utilities/transactions/transaction_lock_mgr.cc

@ -2,6 +2,7 @@
## Unreleased ## Unreleased
### New Features ### New Features
* Add Iterator::Refresh(), which allows users to update the iterator state so that they can avoid some initialization costs of recreating iterators. * Add Iterator::Refresh(), which allows users to update the iterator state so that they can avoid some initialization costs of recreating iterators.
* Replace dynamic_cast<> (except unit test) so people can choose to build with RTTI off. With make, release mode is by default built with -fno-rtti and debug mode is built without it. Users can override it by setting USE_RTTI=0 or 1.
## 5.7.0 (07/13/2017) ## 5.7.0 (07/13/2017)
### Public API Change ### Public API Change

@ -101,7 +101,19 @@ endif
ifeq ($(DEBUG_LEVEL),0) ifeq ($(DEBUG_LEVEL),0)
OPT += -DNDEBUG OPT += -DNDEBUG
DISABLE_WARNING_AS_ERROR=1 DISABLE_WARNING_AS_ERROR=1
ifneq ($(USE_RTTI), 1)
CXXFLAGS += -fno-rtti
else
CXXFLAGS += -DROCKSDB_USE_RTTI
endif
else else
ifneq ($(USE_RTTI), 0)
CXXFLAGS += -DROCKSDB_USE_RTTI
else
CXXFLAGS += -fno-rtti
endif
$(warning Warning: Compiling in debug mode. Don't use the resulting binary in production) $(warning Warning: Compiling in debug mode. Don't use the resulting binary in production)
endif endif

@ -27,6 +27,11 @@ std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
#include <atomic> #include <atomic>
#include <deque> #include <deque>
// "tbb/concurrent_hash_map.h" requires RTTI if exception is enabled.
// Disable it so users can chooose to disable RTTI.
#ifndef ROCKSDB_USE_RTTI
#define TBB_USE_EXCEPTIONS 0
#endif
#include "tbb/concurrent_hash_map.h" #include "tbb/concurrent_hash_map.h"
#include "cache/sharded_cache.h" #include "cache/sharded_cache.h"

@ -9,16 +9,18 @@
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "db/db_impl.h" #include "db/db_impl.h"
#include "util/cast_util.h"
namespace rocksdb { namespace rocksdb {
void CancelAllBackgroundWork(DB* db, bool wait) { void CancelAllBackgroundWork(DB* db, bool wait) {
(dynamic_cast<DBImpl*>(db->GetRootDB()))->CancelAllBackgroundWork(wait); (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
->CancelAllBackgroundWork(wait);
} }
Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family, Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
const Slice* begin, const Slice* end) { const Slice* begin, const Slice* end) {
return (dynamic_cast<DBImpl*>(db->GetRootDB())) return (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
->DeleteFilesInRange(column_family, begin, end); ->DeleteFilesInRange(column_family, begin, end);
} }

@ -235,11 +235,11 @@ class DBImpl : public DB {
ColumnFamilyHandle* column_family, ColumnFamilyHandle* column_family,
ColumnFamilyMetaData* metadata) override; ColumnFamilyMetaData* metadata) override;
// experimental API
Status SuggestCompactRange(ColumnFamilyHandle* column_family, Status SuggestCompactRange(ColumnFamilyHandle* column_family,
const Slice* begin, const Slice* end); const Slice* begin, const Slice* end) override;
Status PromoteL0(ColumnFamilyHandle* column_family, int target_level); Status PromoteL0(ColumnFamilyHandle* column_family,
int target_level) override;
// Similar to Write() but will call the callback once on the single write // Similar to Write() but will call the callback once on the single write
// thread to determine whether it is safe to perform the write. // thread to determine whether it is safe to perform the write.

@ -157,6 +157,9 @@ class InternalKeyComparator : public Comparator {
int Compare(const InternalKey& a, const InternalKey& b) const; int Compare(const InternalKey& a, const InternalKey& b) const;
int Compare(const ParsedInternalKey& a, const ParsedInternalKey& b) const; int Compare(const ParsedInternalKey& a, const ParsedInternalKey& b) const;
virtual const Comparator* GetRootComparator() const override {
return user_comparator_->GetRootComparator();
}
}; };
// Modules in this directory should keep internal keys wrapped inside // Modules in this directory should keep internal keys wrapped inside

@ -14,20 +14,18 @@ namespace experimental {
Status SuggestCompactRange(DB* db, ColumnFamilyHandle* column_family, Status SuggestCompactRange(DB* db, ColumnFamilyHandle* column_family,
const Slice* begin, const Slice* end) { const Slice* begin, const Slice* end) {
auto dbimpl = dynamic_cast<DBImpl*>(db); if (db == nullptr) {
if (dbimpl == nullptr) { return Status::InvalidArgument("DB is empty");
return Status::InvalidArgument("Didn't recognize DB object");
} }
return dbimpl->SuggestCompactRange(column_family, begin, end); return db->SuggestCompactRange(column_family, begin, end);
} }
Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) { Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) {
auto dbimpl = dynamic_cast<DBImpl*>(db); if (db == nullptr) {
if (dbimpl == nullptr) {
return Status::InvalidArgument("Didn't recognize DB object"); return Status::InvalidArgument("Didn't recognize DB object");
} }
return dbimpl->PromoteL0(column_family, target_level); return db->PromoteL0(column_family, target_level);
} }
#else // ROCKSDB_LITE #else // ROCKSDB_LITE

@ -26,6 +26,7 @@
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/options.h" #include "rocksdb/options.h"
#include "rocksdb/write_batch.h" #include "rocksdb/write_batch.h"
#include "util/cast_util.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/file_reader_writer.h" #include "util/file_reader_writer.h"
#include "util/filename.h" #include "util/filename.h"
@ -273,8 +274,8 @@ namespace {
struct CompareLogByPointer { struct CompareLogByPointer {
bool operator()(const std::unique_ptr<LogFile>& a, bool operator()(const std::unique_ptr<LogFile>& a,
const std::unique_ptr<LogFile>& b) { const std::unique_ptr<LogFile>& b) {
LogFileImpl* a_impl = dynamic_cast<LogFileImpl*>(a.get()); LogFileImpl* a_impl = static_cast_with_check<LogFileImpl, LogFile>(a.get());
LogFileImpl* b_impl = dynamic_cast<LogFileImpl*>(b.get()); LogFileImpl* b_impl = static_cast_with_check<LogFileImpl, LogFile>(b.get());
return *a_impl < *b_impl; return *a_impl < *b_impl;
} }
}; };

4
env/mock_env.cc vendored

@ -11,6 +11,7 @@
#include <algorithm> #include <algorithm>
#include <chrono> #include <chrono>
#include "port/sys_time.h" #include "port/sys_time.h"
#include "util/cast_util.h"
#include "util/murmurhash.h" #include "util/murmurhash.h"
#include "util/random.h" #include "util/random.h"
#include "util/rate_limiter.h" #include "util/rate_limiter.h"
@ -711,7 +712,8 @@ Status MockEnv::LockFile(const std::string& fname, FileLock** flock) {
} }
Status MockEnv::UnlockFile(FileLock* flock) { Status MockEnv::UnlockFile(FileLock* flock) {
std::string fn = dynamic_cast<MockEnvFileLock*>(flock)->FileName(); std::string fn =
static_cast_with_check<MockEnvFileLock, FileLock>(flock)->FileName();
{ {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
if (file_map_.find(fn) != file_map_.end()) { if (file_map_.find(fn) != file_map_.end()) {

@ -8,6 +8,10 @@ ifndef DISABLE_JEMALLOC
PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE) PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
endif endif
ifneq ($(USE_RTTI), 1)
CXXFLAGS += -fno-rtti
endif
.PHONY: clean librocksdb .PHONY: clean librocksdb
all: simple_example column_families_example compact_files_example c_simple_example optimistic_transaction_example transaction_example compaction_filter_example options_file_example all: simple_example column_families_example compact_files_example c_simple_example optimistic_transaction_example transaction_example compaction_filter_example options_file_example

@ -59,7 +59,11 @@ int main() {
MyFilter filter; MyFilter filter;
system("rm -rf /tmp/rocksmergetest"); int ret = system("rm -rf /tmp/rocksmergetest");
if (ret != 0) {
fprintf(stderr, "Error deleting /tmp/rocksmergetest, code: %d\n", ret);
return ret;
}
rocksdb::Options options; rocksdb::Options options;
options.create_if_missing = true; options.create_if_missing = true;
options.merge_operator.reset(new MyMerge); options.merge_operator.reset(new MyMerge);

@ -64,6 +64,10 @@ class Comparator {
// Simple comparator implementations may return with *key unchanged, // Simple comparator implementations may return with *key unchanged,
// i.e., an implementation of this method that does nothing is correct. // i.e., an implementation of this method that does nothing is correct.
virtual void FindShortSuccessor(std::string* key) const = 0; virtual void FindShortSuccessor(std::string* key) const = 0;
// if it is a wrapped comparator, may return the root one.
// return itself it is not wrapped.
virtual const Comparator* GetRootComparator() const { return this; }
}; };
// Return a builtin comparator that uses lexicographic byte-wise // Return a builtin comparator that uses lexicographic byte-wise

@ -1097,6 +1097,17 @@ class DB {
virtual Status GetPropertiesOfTablesInRange( virtual Status GetPropertiesOfTablesInRange(
ColumnFamilyHandle* column_family, const Range* range, std::size_t n, ColumnFamilyHandle* column_family, const Range* range, std::size_t n,
TablePropertiesCollection* props) = 0; TablePropertiesCollection* props) = 0;
virtual Status SuggestCompactRange(ColumnFamilyHandle* column_family,
const Slice* begin, const Slice* end) {
return Status::NotSupported("SuggestCompactRange() is not implemented.");
}
virtual Status PromoteL0(ColumnFamilyHandle* column_family,
int target_level) {
return Status::NotSupported("PromoteL0() is not implemented.");
}
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
// Needed for StackableDB // Needed for StackableDB

@ -467,6 +467,12 @@ class TableFactory {
// RocksDB prints configurations at DB Open(). // RocksDB prints configurations at DB Open().
virtual std::string GetPrintableTableOptions() const = 0; virtual std::string GetPrintableTableOptions() const = 0;
virtual Status GetOptionString(std::string* opt_string,
const std::string& delimiter) const {
return Status::NotSupported(
"The table factory doesn't implement GetOptionString().");
}
// Returns the raw pointer of the table options that is used by this // Returns the raw pointer of the table options that is used by this
// TableFactory, or nullptr if this function is not supported. // TableFactory, or nullptr if this function is not supported.
// Since the return value is a raw pointer, the TableFactory owns the // Since the return value is a raw pointer, the TableFactory owns the

@ -350,6 +350,17 @@ class StackableDB : public DB {
return db_->GetUpdatesSince(seq_number, iter, read_options); return db_->GetUpdatesSince(seq_number, iter, read_options);
} }
virtual Status SuggestCompactRange(ColumnFamilyHandle* column_family,
const Slice* begin,
const Slice* end) override {
return db_->SuggestCompactRange(column_family, begin, end);
}
virtual Status PromoteL0(ColumnFamilyHandle* column_family,
int target_level) override {
return db_->PromoteL0(column_family, target_level);
}
virtual ColumnFamilyHandle* DefaultColumnFamily() const override { virtual ColumnFamilyHandle* DefaultColumnFamily() const override {
return db_->DefaultColumnFamily(); return db_->DefaultColumnFamily();
} }

@ -19,6 +19,7 @@
#include <stdio.h> #include <stdio.h>
#include "port/port.h" #include "port/port.h"
#include "util/cast_util.h"
namespace rocksdb { namespace rocksdb {
@ -255,7 +256,8 @@ void HistogramImpl::Add(uint64_t value) {
void HistogramImpl::Merge(const Histogram& other) { void HistogramImpl::Merge(const Histogram& other) {
if (strcmp(Name(), other.Name()) == 0) { if (strcmp(Name(), other.Name()) == 0) {
Merge(dynamic_cast<const HistogramImpl&>(other)); Merge(
*static_cast_with_check<const HistogramImpl, const Histogram>(&other));
} }
} }

@ -9,6 +9,7 @@
#include "monitoring/histogram_windowing.h" #include "monitoring/histogram_windowing.h"
#include "monitoring/histogram.h" #include "monitoring/histogram.h"
#include "util/cast_util.h"
#include <algorithm> #include <algorithm>
@ -64,7 +65,9 @@ void HistogramWindowingImpl::Add(uint64_t value){
void HistogramWindowingImpl::Merge(const Histogram& other) { void HistogramWindowingImpl::Merge(const Histogram& other) {
if (strcmp(Name(), other.Name()) == 0) { if (strcmp(Name(), other.Name()) == 0) {
Merge(dynamic_cast<const HistogramWindowingImpl&>(other)); Merge(
*static_cast_with_check<const HistogramWindowingImpl, const Histogram>(
&other));
} }
} }

@ -21,6 +21,7 @@
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "table/block_based_table_factory.h" #include "table/block_based_table_factory.h"
#include "table/plain_table_factory.h" #include "table/plain_table_factory.h"
#include "util/cast_util.h"
#include "util/string_util.h" #include "util/string_util.h"
namespace rocksdb { namespace rocksdb {
@ -303,6 +304,7 @@ bool ParseSliceTransform(
// SliceTransforms here. // SliceTransforms here.
return false; return false;
} }
} // anonymouse namespace
bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
const std::string& value) { const std::string& value) {
@ -383,8 +385,6 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
return true; return true;
} }
} // anonymouse namespace
bool SerializeSingleOptionHelper(const char* opt_address, bool SerializeSingleOptionHelper(const char* opt_address,
const OptionType opt_type, const OptionType opt_type,
std::string* value) { std::string* value) {
@ -466,12 +466,14 @@ bool SerializeSingleOptionHelper(const char* opt_address,
// Since the user-specified comparator will be wrapped by // Since the user-specified comparator will be wrapped by
// InternalKeyComparator, we should persist the user-specified one // InternalKeyComparator, we should persist the user-specified one
// instead of InternalKeyComparator. // instead of InternalKeyComparator.
const auto* internal_comparator = if (*ptr == nullptr) {
dynamic_cast<const InternalKeyComparator*>(*ptr); *value = kNullptrString;
if (internal_comparator != nullptr) {
*value = internal_comparator->user_comparator()->Name();
} else { } else {
*value = *ptr ? (*ptr)->Name() : kNullptrString; const Comparator* root_comp = (*ptr)->GetRootComparator();
if (root_comp == nullptr) {
root_comp = (*ptr);
}
*value = root_comp->Name();
} }
break; break;
} }
@ -693,7 +695,8 @@ Status ParseColumnFamilyOption(const std::string& name,
if (name == "block_based_table_factory") { if (name == "block_based_table_factory") {
// Nested options // Nested options
BlockBasedTableOptions table_opt, base_table_options; BlockBasedTableOptions table_opt, base_table_options;
auto block_based_table_factory = dynamic_cast<BlockBasedTableFactory*>( BlockBasedTableFactory* block_based_table_factory =
static_cast_with_check<BlockBasedTableFactory, TableFactory>(
new_options->table_factory.get()); new_options->table_factory.get());
if (block_based_table_factory != nullptr) { if (block_based_table_factory != nullptr) {
base_table_options = block_based_table_factory->table_options(); base_table_options = block_based_table_factory->table_options();
@ -708,7 +711,8 @@ Status ParseColumnFamilyOption(const std::string& name,
} else if (name == "plain_table_factory") { } else if (name == "plain_table_factory") {
// Nested options // Nested options
PlainTableOptions table_opt, base_table_options; PlainTableOptions table_opt, base_table_options;
auto plain_table_factory = dynamic_cast<PlainTableFactory*>( PlainTableFactory* plain_table_factory =
static_cast_with_check<PlainTableFactory, TableFactory>(
new_options->table_factory.get()); new_options->table_factory.get());
if (plain_table_factory != nullptr) { if (plain_table_factory != nullptr) {
base_table_options = plain_table_factory->table_options(); base_table_options = plain_table_factory->table_options();
@ -909,59 +913,6 @@ std::vector<CompressionType> GetSupportedCompressions() {
return supported_compressions; return supported_compressions;
} }
bool SerializeSingleBlockBasedTableOption(
std::string* opt_string, const BlockBasedTableOptions& bbt_options,
const std::string& name, const std::string& delimiter) {
auto iter = block_based_table_type_info.find(name);
if (iter == block_based_table_type_info.end()) {
return false;
}
auto& opt_info = iter->second;
const char* opt_address =
reinterpret_cast<const char*>(&bbt_options) + opt_info.offset;
std::string value;
bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value);
if (result) {
*opt_string = name + "=" + value + delimiter;
}
return result;
}
Status GetStringFromBlockBasedTableOptions(
std::string* opt_string, const BlockBasedTableOptions& bbt_options,
const std::string& delimiter) {
assert(opt_string);
opt_string->clear();
for (auto iter = block_based_table_type_info.begin();
iter != block_based_table_type_info.end(); ++iter) {
if (iter->second.verification == OptionVerificationType::kDeprecated) {
// If the option is no longer used in rocksdb and marked as deprecated,
// we skip it in the serialization.
continue;
}
std::string single_output;
bool result = SerializeSingleBlockBasedTableOption(
&single_output, bbt_options, iter->first, delimiter);
assert(result);
if (result) {
opt_string->append(single_output);
}
}
return Status::OK();
}
Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf,
const std::string& delimiter) {
const auto* bbtf = dynamic_cast<const BlockBasedTableFactory*>(tf);
opts_str->clear();
if (bbtf != nullptr) {
return GetStringFromBlockBasedTableOptions(opts_str, bbtf->table_options(),
delimiter);
}
return Status::OK();
}
Status ParseDBOption(const std::string& name, Status ParseDBOption(const std::string& name,
const std::string& org_value, const std::string& org_value,
DBOptions* new_options, DBOptions* new_options,
@ -1003,242 +954,6 @@ Status ParseDBOption(const std::string& name,
return Status::OK(); return Status::OK();
} }
std::string ParseBlockBasedTableOption(const std::string& name,
const std::string& org_value,
BlockBasedTableOptions* new_options,
bool input_strings_escaped = false,
bool ignore_unknown_options = false) {
const std::string& value =
input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
if (!input_strings_escaped) {
// if the input string is not escaped, it means this function is
// invoked from SetOptions, which takes the old format.
if (name == "block_cache") {
new_options->block_cache = NewLRUCache(ParseSizeT(value));
return "";
} else if (name == "block_cache_compressed") {
new_options->block_cache_compressed = NewLRUCache(ParseSizeT(value));
return "";
} else if (name == "filter_policy") {
// Expect the following format
// bloomfilter:int:bool
const std::string kName = "bloomfilter:";
if (value.compare(0, kName.size(), kName) != 0) {
return "Invalid filter policy name";
}
size_t pos = value.find(':', kName.size());
if (pos == std::string::npos) {
return "Invalid filter policy config, missing bits_per_key";
}
int bits_per_key =
ParseInt(trim(value.substr(kName.size(), pos - kName.size())));
bool use_block_based_builder =
ParseBoolean("use_block_based_builder", trim(value.substr(pos + 1)));
new_options->filter_policy.reset(
NewBloomFilterPolicy(bits_per_key, use_block_based_builder));
return "";
}
}
const auto iter = block_based_table_type_info.find(name);
if (iter == block_based_table_type_info.end()) {
if (ignore_unknown_options) {
return "";
} else {
return "Unrecognized option";
}
}
const auto& opt_info = iter->second;
if (opt_info.verification != OptionVerificationType::kDeprecated &&
!ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset,
opt_info.type, value)) {
return "Invalid value";
}
return "";
}
std::string ParsePlainTableOptions(const std::string& name,
const std::string& org_value,
PlainTableOptions* new_options,
bool input_strings_escaped = false,
bool ignore_unknown_options = false) {
const std::string& value =
input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
const auto iter = plain_table_type_info.find(name);
if (iter == plain_table_type_info.end()) {
if (ignore_unknown_options) {
return "";
} else {
return "Unrecognized option";
}
}
const auto& opt_info = iter->second;
if (opt_info.verification != OptionVerificationType::kDeprecated &&
!ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset,
opt_info.type, value)) {
return "Invalid value";
}
return "";
}
Status GetBlockBasedTableOptionsFromMap(
const BlockBasedTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map,
BlockBasedTableOptions* new_table_options, bool input_strings_escaped,
bool ignore_unknown_options) {
assert(new_table_options);
*new_table_options = table_options;
for (const auto& o : opts_map) {
auto error_message = ParseBlockBasedTableOption(
o.first, o.second, new_table_options, input_strings_escaped,
ignore_unknown_options);
if (error_message != "") {
const auto iter = block_based_table_type_info.find(o.first);
if (iter == block_based_table_type_info.end() ||
!input_strings_escaped || // !input_strings_escaped indicates
// the old API, where everything is
// parsable.
(iter->second.verification != OptionVerificationType::kByName &&
iter->second.verification !=
OptionVerificationType::kByNameAllowNull &&
iter->second.verification !=
OptionVerificationType::kDeprecated)) {
// Restore "new_options" to the default "base_options".
*new_table_options = table_options;
return Status::InvalidArgument("Can't parse BlockBasedTableOptions:",
o.first + " " + error_message);
}
}
}
return Status::OK();
}
Status GetBlockBasedTableOptionsFromString(
const BlockBasedTableOptions& table_options,
const std::string& opts_str,
BlockBasedTableOptions* new_table_options) {
std::unordered_map<std::string, std::string> opts_map;
Status s = StringToMap(opts_str, &opts_map);
if (!s.ok()) {
return s;
}
return GetBlockBasedTableOptionsFromMap(table_options, opts_map,
new_table_options);
}
Status GetPlainTableOptionsFromMap(
const PlainTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map,
PlainTableOptions* new_table_options, bool input_strings_escaped,
bool ignore_unknown_options) {
assert(new_table_options);
*new_table_options = table_options;
for (const auto& o : opts_map) {
auto error_message = ParsePlainTableOptions(
o.first, o.second, new_table_options, input_strings_escaped);
if (error_message != "") {
const auto iter = plain_table_type_info.find(o.first);
if (iter == plain_table_type_info.end() ||
!input_strings_escaped || // !input_strings_escaped indicates
// the old API, where everything is
// parsable.
(iter->second.verification != OptionVerificationType::kByName &&
iter->second.verification !=
OptionVerificationType::kByNameAllowNull &&
iter->second.verification !=
OptionVerificationType::kDeprecated)) {
// Restore "new_options" to the default "base_options".
*new_table_options = table_options;
return Status::InvalidArgument("Can't parse PlainTableOptions:",
o.first + " " + error_message);
}
}
}
return Status::OK();
}
Status GetPlainTableOptionsFromString(
const PlainTableOptions& table_options,
const std::string& opts_str,
PlainTableOptions* new_table_options) {
std::unordered_map<std::string, std::string> opts_map;
Status s = StringToMap(opts_str, &opts_map);
if (!s.ok()) {
return s;
}
return GetPlainTableOptionsFromMap(table_options, opts_map,
new_table_options);
}
Status GetMemTableRepFactoryFromString(const std::string& opts_str,
std::unique_ptr<MemTableRepFactory>* new_mem_factory) {
std::vector<std::string> opts_list = StringSplit(opts_str, ':');
size_t len = opts_list.size();
if (opts_list.size() <= 0 || opts_list.size() > 2) {
return Status::InvalidArgument("Can't parse memtable_factory option ",
opts_str);
}
MemTableRepFactory* mem_factory = nullptr;
if (opts_list[0] == "skip_list") {
// Expecting format
// skip_list:<lookahead>
if (2 == len) {
size_t lookahead = ParseSizeT(opts_list[1]);
mem_factory = new SkipListFactory(lookahead);
} else if (1 == len) {
mem_factory = new SkipListFactory();
}
} else if (opts_list[0] == "prefix_hash") {
// Expecting format
// prfix_hash:<hash_bucket_count>
if (2 == len) {
size_t hash_bucket_count = ParseSizeT(opts_list[1]);
mem_factory = NewHashSkipListRepFactory(hash_bucket_count);
} else if (1 == len) {
mem_factory = NewHashSkipListRepFactory();
}
} else if (opts_list[0] == "hash_linkedlist") {
// Expecting format
// hash_linkedlist:<hash_bucket_count>
if (2 == len) {
size_t hash_bucket_count = ParseSizeT(opts_list[1]);
mem_factory = NewHashLinkListRepFactory(hash_bucket_count);
} else if (1 == len) {
mem_factory = NewHashLinkListRepFactory();
}
} else if (opts_list[0] == "vector") {
// Expecting format
// vector:<count>
if (2 == len) {
size_t count = ParseSizeT(opts_list[1]);
mem_factory = new VectorRepFactory(count);
} else if (1 == len) {
mem_factory = new VectorRepFactory();
}
} else if (opts_list[0] == "cuckoo") {
// Expecting format
// cuckoo:<write_buffer_size>
if (2 == len) {
size_t write_buffer_size = ParseSizeT(opts_list[1]);
mem_factory= NewHashCuckooRepFactory(write_buffer_size);
} else if (1 == len) {
return Status::InvalidArgument("Can't parse memtable_factory option ",
opts_str);
}
} else {
return Status::InvalidArgument("Unrecognized memtable_factory option ",
opts_str);
}
if (mem_factory != nullptr){
new_mem_factory->reset(mem_factory);
}
return Status::OK();
}
Status GetColumnFamilyOptionsFromMap( Status GetColumnFamilyOptionsFromMap(
const ColumnFamilyOptions& base_options, const ColumnFamilyOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,

@ -60,9 +60,6 @@ Status GetTableFactoryFromMap(
std::shared_ptr<TableFactory>* table_factory, std::shared_ptr<TableFactory>* table_factory,
bool ignore_unknown_options = false); bool ignore_unknown_options = false);
Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf,
const std::string& delimiter = "; ");
enum class OptionType { enum class OptionType {
kBoolean, kBoolean,
kInt, kInt,
@ -580,109 +577,6 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
{offset_of(&ColumnFamilyOptions::compaction_pri), {offset_of(&ColumnFamilyOptions::compaction_pri),
OptionType::kCompactionPri, OptionVerificationType::kNormal, false, 0}}}; OptionType::kCompactionPri, OptionVerificationType::kNormal, false, 0}}};
static std::unordered_map<std::string, OptionTypeInfo>
block_based_table_type_info = {
/* currently not supported
std::shared_ptr<Cache> block_cache = nullptr;
std::shared_ptr<Cache> block_cache_compressed = nullptr;
*/
{"flush_block_policy_factory",
{offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName,
false, 0}},
{"cache_index_and_filter_blocks",
{offsetof(struct BlockBasedTableOptions,
cache_index_and_filter_blocks),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"cache_index_and_filter_blocks_with_high_priority",
{offsetof(struct BlockBasedTableOptions,
cache_index_and_filter_blocks_with_high_priority),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"pin_l0_filter_and_index_blocks_in_cache",
{offsetof(struct BlockBasedTableOptions,
pin_l0_filter_and_index_blocks_in_cache),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"index_type",
{offsetof(struct BlockBasedTableOptions, index_type),
OptionType::kBlockBasedTableIndexType,
OptionVerificationType::kNormal, false, 0}},
{"hash_index_allow_collision",
{offsetof(struct BlockBasedTableOptions, hash_index_allow_collision),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"checksum",
{offsetof(struct BlockBasedTableOptions, checksum),
OptionType::kChecksumType, OptionVerificationType::kNormal, false,
0}},
{"no_block_cache",
{offsetof(struct BlockBasedTableOptions, no_block_cache),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"block_size",
{offsetof(struct BlockBasedTableOptions, block_size),
OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
{"block_size_deviation",
{offsetof(struct BlockBasedTableOptions, block_size_deviation),
OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
{"block_restart_interval",
{offsetof(struct BlockBasedTableOptions, block_restart_interval),
OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
{"index_block_restart_interval",
{offsetof(struct BlockBasedTableOptions, index_block_restart_interval),
OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
{"index_per_partition",
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, false,
0}},
{"metadata_block_size",
{offsetof(struct BlockBasedTableOptions, metadata_block_size),
OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
{"partition_filters",
{offsetof(struct BlockBasedTableOptions, partition_filters),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"filter_policy",
{offsetof(struct BlockBasedTableOptions, filter_policy),
OptionType::kFilterPolicy, OptionVerificationType::kByName, false,
0}},
{"whole_key_filtering",
{offsetof(struct BlockBasedTableOptions, whole_key_filtering),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"skip_table_builder_flush",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
0}},
{"format_version",
{offsetof(struct BlockBasedTableOptions, format_version),
OptionType::kUInt32T, OptionVerificationType::kNormal, false, 0}},
{"verify_compression",
{offsetof(struct BlockBasedTableOptions, verify_compression),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"read_amp_bytes_per_bit",
{offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit),
OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}};
static std::unordered_map<std::string, OptionTypeInfo> plain_table_type_info = {
{"user_key_len",
{offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T,
OptionVerificationType::kNormal, false, 0}},
{"bloom_bits_per_key",
{offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt,
OptionVerificationType::kNormal, false, 0}},
{"hash_table_ratio",
{offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble,
OptionVerificationType::kNormal, false, 0}},
{"index_sparseness",
{offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT,
OptionVerificationType::kNormal, false, 0}},
{"huge_page_tlb_size",
{offsetof(struct PlainTableOptions, huge_page_tlb_size),
OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
{"encoding_type",
{offsetof(struct PlainTableOptions, encoding_type),
OptionType::kEncodingType, OptionVerificationType::kByName, false, 0}},
{"full_scan_mode",
{offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean,
OptionVerificationType::kNormal, false, 0}},
{"store_index_in_file",
{offsetof(struct PlainTableOptions, store_index_in_file),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}};
static std::unordered_map<std::string, CompressionType> static std::unordered_map<std::string, CompressionType>
compression_type_string_map = { compression_type_string_map = {
{"kNoCompression", kNoCompression}, {"kNoCompression", kNoCompression},
@ -745,6 +639,12 @@ static std::unordered_map<std::string, InfoLogLevel> info_log_level_string_map =
{"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL}, {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL},
{"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}}; {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}};
extern Status StringToMap(
const std::string& opts_str,
std::unordered_map<std::string, std::string>* opts_map);
extern bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
const std::string& value);
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
} // namespace rocksdb } // namespace rocksdb

@ -16,6 +16,7 @@
#include "options/options_helper.h" #include "options/options_helper.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "util/cast_util.h"
#include "util/string_util.h" #include "util/string_util.h"
#include "util/sync_point.h" #include "util/sync_point.h"
@ -84,7 +85,8 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
writable->Append("[" + opt_section_titles[kOptionSectionTableOptions] + writable->Append("[" + opt_section_titles[kOptionSectionTableOptions] +
tf->Name() + " \"" + EscapeOptionString(cf_names[i]) + tf->Name() + " \"" + EscapeOptionString(cf_names[i]) +
"\"]\n "); "\"]\n ");
s = GetStringFromTableFactory(&options_file_content, tf, "\n "); options_file_content.clear();
s = tf->GetOptionString(&options_file_content, "\n ");
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -507,6 +509,7 @@ namespace {
bool AreEqualDoubles(const double a, const double b) { bool AreEqualDoubles(const double a, const double b) {
return (fabs(a - b) < 0.00001); return (fabs(a - b) < 0.00001);
} }
} // namespace
bool AreEqualOptions( bool AreEqualOptions(
const char* opt1, const char* opt2, const OptionTypeInfo& type_info, const char* opt1, const char* opt2, const OptionTypeInfo& type_info,
@ -613,8 +616,6 @@ bool AreEqualOptions(
} }
} }
} // namespace
Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
const DBOptions& db_opt, const std::vector<std::string>& cf_names, const DBOptions& db_opt, const std::vector<std::string>& cf_names,
const std::vector<ColumnFamilyOptions>& cf_opts, const std::vector<ColumnFamilyOptions>& cf_opts,
@ -762,59 +763,23 @@ Status RocksDBOptionsParser::VerifyCFOptions(
return Status::OK(); return Status::OK();
} }
Status RocksDBOptionsParser::VerifyBlockBasedTableFactory(
const BlockBasedTableFactory* base_tf,
const BlockBasedTableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level) {
if ((base_tf != nullptr) != (file_tf != nullptr) &&
sanity_check_level > kSanityLevelNone) {
return Status::Corruption(
"[RocksDBOptionsParser]: Inconsistent TableFactory class type");
}
if (base_tf == nullptr) {
return Status::OK();
}
assert(file_tf != nullptr);
const auto& base_opt = base_tf->table_options();
const auto& file_opt = file_tf->table_options();
for (auto& pair : block_based_table_type_info) {
if (pair.second.verification == OptionVerificationType::kDeprecated) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
continue;
}
if (BBTOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&file_opt),
pair.second, pair.first, nullptr)) {
return Status::Corruption(
"[RocksDBOptionsParser]: "
"failed the verification on BlockBasedTableOptions::",
pair.first);
}
}
}
return Status::OK();
}
Status RocksDBOptionsParser::VerifyTableFactory( Status RocksDBOptionsParser::VerifyTableFactory(
const TableFactory* base_tf, const TableFactory* file_tf, const TableFactory* base_tf, const TableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level) { OptionsSanityCheckLevel sanity_check_level) {
if (base_tf && file_tf) { if (base_tf && file_tf) {
if (sanity_check_level > kSanityLevelNone && if (sanity_check_level > kSanityLevelNone &&
base_tf->Name() != file_tf->Name()) { std::string(base_tf->Name()) != std::string(file_tf->Name())) {
return Status::Corruption( return Status::Corruption(
"[RocksDBOptionsParser]: " "[RocksDBOptionsParser]: "
"failed the verification on TableFactory->Name()"); "failed the verification on TableFactory->Name()");
} }
auto s = VerifyBlockBasedTableFactory( if (base_tf->Name() == BlockBasedTableFactory::kName) {
dynamic_cast<const BlockBasedTableFactory*>(base_tf), return VerifyBlockBasedTableFactory(
dynamic_cast<const BlockBasedTableFactory*>(file_tf), static_cast_with_check<const BlockBasedTableFactory,
const TableFactory>(base_tf),
static_cast_with_check<const BlockBasedTableFactory,
const TableFactory>(file_tf),
sanity_check_level); sanity_check_level);
if (!s.ok()) {
return s;
} }
// TODO(yhchiang): add checks for other table factory types // TODO(yhchiang): add checks for other table factory types
} else { } else {

@ -38,6 +38,11 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
const std::vector<ColumnFamilyOptions>& cf_opts, const std::vector<ColumnFamilyOptions>& cf_opts,
const std::string& file_name, Env* env); const std::string& file_name, Env* env);
extern bool AreEqualOptions(
const char* opt1, const char* opt2, const OptionTypeInfo& type_info,
const std::string& opt_name,
const std::unordered_map<std::string, std::string>* opt_map);
class RocksDBOptionsParser { class RocksDBOptionsParser {
public: public:
explicit RocksDBOptionsParser(); explicit RocksDBOptionsParser();
@ -86,11 +91,6 @@ class RocksDBOptionsParser {
const TableFactory* base_tf, const TableFactory* file_tf, const TableFactory* base_tf, const TableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
static Status VerifyBlockBasedTableFactory(
const BlockBasedTableFactory* base_tf,
const BlockBasedTableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level);
static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser); static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser);
protected: protected:

@ -889,11 +889,11 @@ TEST_F(OptionsTest, ConvertOptionsTest) {
ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files); ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files);
ASSERT_EQ(converted_opt.compression, leveldb_opt.compression); ASSERT_EQ(converted_opt.compression, leveldb_opt.compression);
std::shared_ptr<BlockBasedTableFactory> table_factory = std::shared_ptr<TableFactory> tb_guard = converted_opt.table_factory;
std::dynamic_pointer_cast<BlockBasedTableFactory>( BlockBasedTableFactory* table_factory =
converted_opt.table_factory); dynamic_cast<BlockBasedTableFactory*>(converted_opt.table_factory.get());
ASSERT_TRUE(table_factory.get() != nullptr); ASSERT_TRUE(table_factory != nullptr);
const BlockBasedTableOptions table_opt = table_factory->table_options(); const BlockBasedTableOptions table_opt = table_factory->table_options();
@ -1278,6 +1278,11 @@ TEST_F(OptionsParserTest, DumpAndParse) {
Random rnd(302); Random rnd(302);
test::RandomInitDBOptions(&base_db_opt, &rnd); test::RandomInitDBOptions(&base_db_opt, &rnd);
base_db_opt.db_log_dir += "/#odd #but #could #happen #path #/\\\\#OMG"; base_db_opt.db_log_dir += "/#odd #but #could #happen #path #/\\\\#OMG";
BlockBasedTableOptions special_bbto;
special_bbto.cache_index_and_filter_blocks = true;
special_bbto.block_size = 999999;
for (int c = 0; c < num_cf; ++c) { for (int c = 0; c < num_cf; ++c) {
ColumnFamilyOptions cf_opt; ColumnFamilyOptions cf_opt;
Random cf_rnd(0xFB + c); Random cf_rnd(0xFB + c);
@ -1287,6 +1292,8 @@ TEST_F(OptionsParserTest, DumpAndParse) {
} }
if (c < 3) { if (c < 3) {
cf_opt.table_factory.reset(test::RandomTableFactory(&rnd, c)); cf_opt.table_factory.reset(test::RandomTableFactory(&rnd, c));
} else if (c == 4) {
cf_opt.table_factory.reset(NewBlockBasedTableFactory(special_bbto));
} }
base_cf_opts.emplace_back(cf_opt); base_cf_opts.emplace_back(cf_opt);
} }
@ -1298,6 +1305,15 @@ TEST_F(OptionsParserTest, DumpAndParse) {
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
ASSERT_OK(parser.Parse(kOptionsFileName, env_.get())); ASSERT_OK(parser.Parse(kOptionsFileName, env_.get()));
// Make sure block-based table factory options was deserialized correctly
std::shared_ptr<TableFactory> ttf = (*parser.cf_opts())[4].table_factory;
ASSERT_EQ(BlockBasedTableFactory::kName, std::string(ttf->Name()));
const BlockBasedTableOptions& parsed_bbto =
static_cast<BlockBasedTableFactory*>(ttf.get())->table_options();
ASSERT_EQ(special_bbto.block_size, parsed_bbto.block_size);
ASSERT_EQ(special_bbto.cache_index_and_filter_blocks,
parsed_bbto.cache_index_and_filter_blocks);
ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
base_db_opt, cf_names, base_cf_opts, kOptionsFileName, env_.get())); base_db_opt, cf_names, base_cf_opts, kOptionsFileName, env_.get()));

@ -13,12 +13,15 @@
#include <string> #include <string>
#include <stdint.h> #include <stdint.h>
#include "options/options_helper.h"
#include "port/port.h" #include "port/port.h"
#include "rocksdb/flush_block_policy.h"
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "rocksdb/convenience.h"
#include "rocksdb/flush_block_policy.h"
#include "table/block_based_table_builder.h" #include "table/block_based_table_builder.h"
#include "table/block_based_table_reader.h" #include "table/block_based_table_reader.h"
#include "table/format.h" #include "table/format.h"
#include "util/string_util.h"
namespace rocksdb { namespace rocksdb {
@ -201,15 +204,203 @@ std::string BlockBasedTableFactory::GetPrintableTableOptions() const {
return ret; return ret;
} }
#ifndef ROCKSDB_LITE
namespace {
bool SerializeSingleBlockBasedTableOption(
std::string* opt_string, const BlockBasedTableOptions& bbt_options,
const std::string& name, const std::string& delimiter) {
auto iter = block_based_table_type_info.find(name);
if (iter == block_based_table_type_info.end()) {
return false;
}
auto& opt_info = iter->second;
const char* opt_address =
reinterpret_cast<const char*>(&bbt_options) + opt_info.offset;
std::string value;
bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value);
if (result) {
*opt_string = name + "=" + value + delimiter;
}
return result;
}
} // namespace
Status BlockBasedTableFactory::GetOptionString(
std::string* opt_string, const std::string& delimiter) const {
assert(opt_string);
opt_string->clear();
for (auto iter = block_based_table_type_info.begin();
iter != block_based_table_type_info.end(); ++iter) {
if (iter->second.verification == OptionVerificationType::kDeprecated) {
// If the option is no longer used in rocksdb and marked as deprecated,
// we skip it in the serialization.
continue;
}
std::string single_output;
bool result = SerializeSingleBlockBasedTableOption(
&single_output, table_options_, iter->first, delimiter);
assert(result);
if (result) {
opt_string->append(single_output);
}
}
return Status::OK();
}
#else
Status BlockBasedTableFactory::GetOptionString(
std::string* opt_string, const std::string& delimiter) const {
return Status::OK();
}
#endif // !ROCKSDB_LITE
const BlockBasedTableOptions& BlockBasedTableFactory::table_options() const { const BlockBasedTableOptions& BlockBasedTableFactory::table_options() const {
return table_options_; return table_options_;
} }
#ifndef ROCKSDB_LITE
namespace {
std::string ParseBlockBasedTableOption(const std::string& name,
const std::string& org_value,
BlockBasedTableOptions* new_options,
bool input_strings_escaped = false,
bool ignore_unknown_options = false) {
const std::string& value =
input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
if (!input_strings_escaped) {
// if the input string is not escaped, it means this function is
// invoked from SetOptions, which takes the old format.
if (name == "block_cache") {
new_options->block_cache = NewLRUCache(ParseSizeT(value));
return "";
} else if (name == "block_cache_compressed") {
new_options->block_cache_compressed = NewLRUCache(ParseSizeT(value));
return "";
} else if (name == "filter_policy") {
// Expect the following format
// bloomfilter:int:bool
const std::string kName = "bloomfilter:";
if (value.compare(0, kName.size(), kName) != 0) {
return "Invalid filter policy name";
}
size_t pos = value.find(':', kName.size());
if (pos == std::string::npos) {
return "Invalid filter policy config, missing bits_per_key";
}
int bits_per_key =
ParseInt(trim(value.substr(kName.size(), pos - kName.size())));
bool use_block_based_builder =
ParseBoolean("use_block_based_builder", trim(value.substr(pos + 1)));
new_options->filter_policy.reset(
NewBloomFilterPolicy(bits_per_key, use_block_based_builder));
return "";
}
}
const auto iter = block_based_table_type_info.find(name);
if (iter == block_based_table_type_info.end()) {
if (ignore_unknown_options) {
return "";
} else {
return "Unrecognized option";
}
}
const auto& opt_info = iter->second;
if (opt_info.verification != OptionVerificationType::kDeprecated &&
!ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset,
opt_info.type, value)) {
return "Invalid value";
}
return "";
}
} // namespace
Status GetBlockBasedTableOptionsFromString(
const BlockBasedTableOptions& table_options, const std::string& opts_str,
BlockBasedTableOptions* new_table_options) {
std::unordered_map<std::string, std::string> opts_map;
Status s = StringToMap(opts_str, &opts_map);
if (!s.ok()) {
return s;
}
return GetBlockBasedTableOptionsFromMap(table_options, opts_map,
new_table_options);
}
Status GetBlockBasedTableOptionsFromMap(
const BlockBasedTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map,
BlockBasedTableOptions* new_table_options, bool input_strings_escaped,
bool ignore_unknown_options) {
assert(new_table_options);
*new_table_options = table_options;
for (const auto& o : opts_map) {
auto error_message = ParseBlockBasedTableOption(
o.first, o.second, new_table_options, input_strings_escaped,
ignore_unknown_options);
if (error_message != "") {
const auto iter = block_based_table_type_info.find(o.first);
if (iter == block_based_table_type_info.end() ||
!input_strings_escaped || // !input_strings_escaped indicates
// the old API, where everything is
// parsable.
(iter->second.verification != OptionVerificationType::kByName &&
iter->second.verification !=
OptionVerificationType::kByNameAllowNull &&
iter->second.verification != OptionVerificationType::kDeprecated)) {
// Restore "new_options" to the default "base_options".
*new_table_options = table_options;
return Status::InvalidArgument("Can't parse BlockBasedTableOptions:",
o.first + " " + error_message);
}
}
}
return Status::OK();
}
Status VerifyBlockBasedTableFactory(
const BlockBasedTableFactory* base_tf,
const BlockBasedTableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level) {
if ((base_tf != nullptr) != (file_tf != nullptr) &&
sanity_check_level > kSanityLevelNone) {
return Status::Corruption(
"[RocksDBOptionsParser]: Inconsistent TableFactory class type");
}
if (base_tf == nullptr) {
return Status::OK();
}
assert(file_tf != nullptr);
const auto& base_opt = base_tf->table_options();
const auto& file_opt = file_tf->table_options();
for (auto& pair : block_based_table_type_info) {
if (pair.second.verification == OptionVerificationType::kDeprecated) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
continue;
}
if (BBTOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&file_opt),
pair.second, pair.first, nullptr)) {
return Status::Corruption(
"[RocksDBOptionsParser]: "
"failed the verification on BlockBasedTableOptions::",
pair.first);
}
}
}
return Status::OK();
}
#endif // !ROCKSDB_LITE
TableFactory* NewBlockBasedTableFactory( TableFactory* NewBlockBasedTableFactory(
const BlockBasedTableOptions& _table_options) { const BlockBasedTableOptions& _table_options) {
return new BlockBasedTableFactory(_table_options); return new BlockBasedTableFactory(_table_options);
} }
const std::string BlockBasedTableFactory::kName = "BlockBasedTable";
const std::string BlockBasedTablePropertyNames::kIndexType = const std::string BlockBasedTablePropertyNames::kIndexType =
"rocksdb.block.based.table.index.type"; "rocksdb.block.based.table.index.type";
const std::string BlockBasedTablePropertyNames::kWholeKeyFiltering = const std::string BlockBasedTablePropertyNames::kWholeKeyFiltering =

@ -13,9 +13,11 @@
#include <memory> #include <memory>
#include <string> #include <string>
#include "db/dbformat.h"
#include "options/options_helper.h"
#include "options/options_parser.h"
#include "rocksdb/flush_block_policy.h" #include "rocksdb/flush_block_policy.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "db/dbformat.h"
namespace rocksdb { namespace rocksdb {
@ -31,7 +33,7 @@ class BlockBasedTableFactory : public TableFactory {
~BlockBasedTableFactory() {} ~BlockBasedTableFactory() {}
const char* Name() const override { return "BlockBasedTable"; } const char* Name() const override { return kName.c_str(); }
Status NewTableReader( Status NewTableReader(
const TableReaderOptions& table_reader_options, const TableReaderOptions& table_reader_options,
@ -49,12 +51,17 @@ class BlockBasedTableFactory : public TableFactory {
std::string GetPrintableTableOptions() const override; std::string GetPrintableTableOptions() const override;
Status GetOptionString(std::string* opt_string,
const std::string& delimiter) const override;
const BlockBasedTableOptions& table_options() const; const BlockBasedTableOptions& table_options() const;
void* GetOptions() override { return &table_options_; } void* GetOptions() override { return &table_options_; }
bool IsDeleteRangeSupported() const override { return true; } bool IsDeleteRangeSupported() const override { return true; }
static const std::string kName;
private: private:
BlockBasedTableOptions table_options_; BlockBasedTableOptions table_options_;
}; };
@ -64,4 +71,87 @@ extern const std::string kHashIndexPrefixesMetadataBlock;
extern const std::string kPropTrue; extern const std::string kPropTrue;
extern const std::string kPropFalse; extern const std::string kPropFalse;
#ifndef ROCKSDB_LITE
extern Status VerifyBlockBasedTableFactory(
const BlockBasedTableFactory* base_tf,
const BlockBasedTableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level);
static std::unordered_map<std::string, OptionTypeInfo>
block_based_table_type_info = {
/* currently not supported
std::shared_ptr<Cache> block_cache = nullptr;
std::shared_ptr<Cache> block_cache_compressed = nullptr;
*/
{"flush_block_policy_factory",
{offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName,
false, 0}},
{"cache_index_and_filter_blocks",
{offsetof(struct BlockBasedTableOptions,
cache_index_and_filter_blocks),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"cache_index_and_filter_blocks_with_high_priority",
{offsetof(struct BlockBasedTableOptions,
cache_index_and_filter_blocks_with_high_priority),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"pin_l0_filter_and_index_blocks_in_cache",
{offsetof(struct BlockBasedTableOptions,
pin_l0_filter_and_index_blocks_in_cache),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"index_type",
{offsetof(struct BlockBasedTableOptions, index_type),
OptionType::kBlockBasedTableIndexType,
OptionVerificationType::kNormal, false, 0}},
{"hash_index_allow_collision",
{offsetof(struct BlockBasedTableOptions, hash_index_allow_collision),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"checksum",
{offsetof(struct BlockBasedTableOptions, checksum),
OptionType::kChecksumType, OptionVerificationType::kNormal, false,
0}},
{"no_block_cache",
{offsetof(struct BlockBasedTableOptions, no_block_cache),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"block_size",
{offsetof(struct BlockBasedTableOptions, block_size),
OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
{"block_size_deviation",
{offsetof(struct BlockBasedTableOptions, block_size_deviation),
OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
{"block_restart_interval",
{offsetof(struct BlockBasedTableOptions, block_restart_interval),
OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
{"index_block_restart_interval",
{offsetof(struct BlockBasedTableOptions, index_block_restart_interval),
OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
{"index_per_partition",
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, false,
0}},
{"metadata_block_size",
{offsetof(struct BlockBasedTableOptions, metadata_block_size),
OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
{"partition_filters",
{offsetof(struct BlockBasedTableOptions, partition_filters),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"filter_policy",
{offsetof(struct BlockBasedTableOptions, filter_policy),
OptionType::kFilterPolicy, OptionVerificationType::kByName, false,
0}},
{"whole_key_filtering",
{offsetof(struct BlockBasedTableOptions, whole_key_filtering),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"skip_table_builder_flush",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
0}},
{"format_version",
{offsetof(struct BlockBasedTableOptions, format_version),
OptionType::kUInt32T, OptionVerificationType::kNormal, false, 0}},
{"verify_compression",
{offsetof(struct BlockBasedTableOptions, verify_compression),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"read_amp_bytes_per_bit",
{offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit),
OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}};
#endif // !ROCKSDB_LITE
} // namespace rocksdb } // namespace rocksdb

@ -76,6 +76,11 @@ class CuckooTableFactory : public TableFactory {
void* GetOptions() override { return &table_options_; } void* GetOptions() override { return &table_options_; }
Status GetOptionString(std::string* opt_string,
const std::string& delimiter) const override {
return Status::OK();
}
private: private:
CuckooTableOptions table_options_; CuckooTableOptions table_options_;
}; };

@ -5,12 +5,15 @@
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#include "table/plain_table_factory.h" #include "table/plain_table_factory.h"
#include <memory>
#include <stdint.h> #include <stdint.h>
#include <memory>
#include "db/dbformat.h" #include "db/dbformat.h"
#include "options/options_helper.h"
#include "port/port.h"
#include "rocksdb/convenience.h"
#include "table/plain_table_builder.h" #include "table/plain_table_builder.h"
#include "table/plain_table_reader.h" #include "table/plain_table_reader.h"
#include "port/port.h" #include "util/string_util.h"
namespace rocksdb { namespace rocksdb {
@ -81,6 +84,143 @@ const PlainTableOptions& PlainTableFactory::table_options() const {
return table_options_; return table_options_;
} }
Status GetPlainTableOptionsFromString(const PlainTableOptions& table_options,
const std::string& opts_str,
PlainTableOptions* new_table_options) {
std::unordered_map<std::string, std::string> opts_map;
Status s = StringToMap(opts_str, &opts_map);
if (!s.ok()) {
return s;
}
return GetPlainTableOptionsFromMap(table_options, opts_map,
new_table_options);
}
Status GetMemTableRepFactoryFromString(
const std::string& opts_str,
std::unique_ptr<MemTableRepFactory>* new_mem_factory) {
std::vector<std::string> opts_list = StringSplit(opts_str, ':');
size_t len = opts_list.size();
if (opts_list.size() <= 0 || opts_list.size() > 2) {
return Status::InvalidArgument("Can't parse memtable_factory option ",
opts_str);
}
MemTableRepFactory* mem_factory = nullptr;
if (opts_list[0] == "skip_list") {
// Expecting format
// skip_list:<lookahead>
if (2 == len) {
size_t lookahead = ParseSizeT(opts_list[1]);
mem_factory = new SkipListFactory(lookahead);
} else if (1 == len) {
mem_factory = new SkipListFactory();
}
} else if (opts_list[0] == "prefix_hash") {
// Expecting format
// prfix_hash:<hash_bucket_count>
if (2 == len) {
size_t hash_bucket_count = ParseSizeT(opts_list[1]);
mem_factory = NewHashSkipListRepFactory(hash_bucket_count);
} else if (1 == len) {
mem_factory = NewHashSkipListRepFactory();
}
} else if (opts_list[0] == "hash_linkedlist") {
// Expecting format
// hash_linkedlist:<hash_bucket_count>
if (2 == len) {
size_t hash_bucket_count = ParseSizeT(opts_list[1]);
mem_factory = NewHashLinkListRepFactory(hash_bucket_count);
} else if (1 == len) {
mem_factory = NewHashLinkListRepFactory();
}
} else if (opts_list[0] == "vector") {
// Expecting format
// vector:<count>
if (2 == len) {
size_t count = ParseSizeT(opts_list[1]);
mem_factory = new VectorRepFactory(count);
} else if (1 == len) {
mem_factory = new VectorRepFactory();
}
} else if (opts_list[0] == "cuckoo") {
// Expecting format
// cuckoo:<write_buffer_size>
if (2 == len) {
size_t write_buffer_size = ParseSizeT(opts_list[1]);
mem_factory = NewHashCuckooRepFactory(write_buffer_size);
} else if (1 == len) {
return Status::InvalidArgument("Can't parse memtable_factory option ",
opts_str);
}
} else {
return Status::InvalidArgument("Unrecognized memtable_factory option ",
opts_str);
}
if (mem_factory != nullptr) {
new_mem_factory->reset(mem_factory);
}
return Status::OK();
}
std::string ParsePlainTableOptions(const std::string& name,
const std::string& org_value,
PlainTableOptions* new_options,
bool input_strings_escaped = false,
bool ignore_unknown_options = false) {
const std::string& value =
input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
const auto iter = plain_table_type_info.find(name);
if (iter == plain_table_type_info.end()) {
if (ignore_unknown_options) {
return "";
} else {
return "Unrecognized option";
}
}
const auto& opt_info = iter->second;
if (opt_info.verification != OptionVerificationType::kDeprecated &&
!ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset,
opt_info.type, value)) {
return "Invalid value";
}
return "";
}
Status GetPlainTableOptionsFromMap(
const PlainTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map,
PlainTableOptions* new_table_options, bool input_strings_escaped,
bool ignore_unknown_options) {
assert(new_table_options);
*new_table_options = table_options;
for (const auto& o : opts_map) {
auto error_message = ParsePlainTableOptions(
o.first, o.second, new_table_options, input_strings_escaped);
if (error_message != "") {
const auto iter = plain_table_type_info.find(o.first);
if (iter == plain_table_type_info.end() ||
!input_strings_escaped || // !input_strings_escaped indicates
// the old API, where everything is
// parsable.
(iter->second.verification != OptionVerificationType::kByName &&
iter->second.verification !=
OptionVerificationType::kByNameAllowNull &&
iter->second.verification != OptionVerificationType::kDeprecated)) {
// Restore "new_options" to the default "base_options".
*new_table_options = table_options;
return Status::InvalidArgument("Can't parse PlainTableOptions:",
o.first + " " + error_message);
}
}
}
return Status::OK();
}
extern TableFactory* NewPlainTableFactory(const PlainTableOptions& options) { extern TableFactory* NewPlainTableFactory(const PlainTableOptions& options) {
return new PlainTableFactory(options); return new PlainTableFactory(options);
} }

@ -9,6 +9,7 @@
#include <string> #include <string>
#include <stdint.h> #include <stdint.h>
#include "options/options_helper.h"
#include "rocksdb/options.h" #include "rocksdb/options.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
@ -170,9 +171,40 @@ class PlainTableFactory : public TableFactory {
void* GetOptions() override { return &table_options_; } void* GetOptions() override { return &table_options_; }
Status GetOptionString(std::string* opt_string,
const std::string& delimiter) const override {
return Status::OK();
}
private: private:
PlainTableOptions table_options_; PlainTableOptions table_options_;
}; };
static std::unordered_map<std::string, OptionTypeInfo> plain_table_type_info = {
{"user_key_len",
{offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T,
OptionVerificationType::kNormal, false, 0}},
{"bloom_bits_per_key",
{offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt,
OptionVerificationType::kNormal, false, 0}},
{"hash_table_ratio",
{offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble,
OptionVerificationType::kNormal, false, 0}},
{"index_sparseness",
{offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT,
OptionVerificationType::kNormal, false, 0}},
{"huge_page_tlb_size",
{offsetof(struct PlainTableOptions, huge_page_tlb_size),
OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
{"encoding_type",
{offsetof(struct PlainTableOptions, encoding_type),
OptionType::kEncodingType, OptionVerificationType::kByName, false, 0}},
{"full_scan_mode",
{offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean,
OptionVerificationType::kNormal, false, 0}},
{"store_index_in_file",
{offsetof(struct PlainTableOptions, store_index_in_file),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}};
} // namespace rocksdb } // namespace rocksdb
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE

@ -28,6 +28,7 @@
#include <atomic> #include <atomic>
#include <condition_variable> #include <condition_variable>
#include <cstddef> #include <cstddef>
#include <memory>
#include <mutex> #include <mutex>
#include <thread> #include <thread>
#include <unordered_map> #include <unordered_map>
@ -57,6 +58,7 @@
#include "rocksdb/utilities/transaction.h" #include "rocksdb/utilities/transaction.h"
#include "rocksdb/utilities/transaction_db.h" #include "rocksdb/utilities/transaction_db.h"
#include "rocksdb/write_batch.h" #include "rocksdb/write_batch.h"
#include "util/cast_util.h"
#include "util/compression.h" #include "util/compression.h"
#include "util/crc32c.h" #include "util/crc32c.h"
#include "util/mutexlock.h" #include "util/mutexlock.h"
@ -2551,7 +2553,9 @@ void VerifyDBFromDB(std::string& truth_db_name) {
} }
if (FLAGS_simcache_size >= 0) { if (FLAGS_simcache_size >= 0) {
fprintf(stdout, "SIMULATOR CACHE STATISTICS:\n%s\n", fprintf(stdout, "SIMULATOR CACHE STATISTICS:\n%s\n",
std::dynamic_pointer_cast<SimCache>(cache_)->ToString().c_str()); static_cast_with_check<SimCache, Cache>(cache_.get())
->ToString()
.c_str());
} }
} }

@ -29,6 +29,7 @@
#include "table/scoped_arena_iterator.h" #include "table/scoped_arena_iterator.h"
#include "tools/ldb_cmd_impl.h" #include "tools/ldb_cmd_impl.h"
#include "tools/sst_dump_tool_imp.h" #include "tools/sst_dump_tool_imp.h"
#include "util/cast_util.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/filename.h" #include "util/filename.h"
#include "util/stderr_logger.h" #include "util/stderr_logger.h"
@ -1493,8 +1494,7 @@ void DBDumperCommand::DoDumpCommand() {
if (max_keys == 0) if (max_keys == 0)
break; break;
if (is_db_ttl_) { if (is_db_ttl_) {
TtlIterator* it_ttl = dynamic_cast<TtlIterator*>(iter); TtlIterator* it_ttl = static_cast_with_check<TtlIterator, Iterator>(iter);
assert(it_ttl);
rawtime = it_ttl->timestamp(); rawtime = it_ttl->timestamp();
if (rawtime < ttl_start || rawtime >= ttl_end) { if (rawtime < ttl_start || rawtime >= ttl_end) {
continue; continue;
@ -2291,8 +2291,7 @@ void ScanCommand::DoCommand() {
it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_); it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_);
it->Next()) { it->Next()) {
if (is_db_ttl_) { if (is_db_ttl_) {
TtlIterator* it_ttl = dynamic_cast<TtlIterator*>(it); TtlIterator* it_ttl = static_cast_with_check<TtlIterator, Iterator>(it);
assert(it_ttl);
int rawtime = it_ttl->timestamp(); int rawtime = it_ttl->timestamp();
if (rawtime < ttl_start || rawtime >= ttl_end) { if (rawtime < ttl_start || rawtime >= ttl_end) {
continue; continue;

@ -14,6 +14,7 @@
#include <inttypes.h> #include <inttypes.h>
#include <iostream> #include <iostream>
#include <map> #include <map>
#include <memory>
#include <sstream> #include <sstream>
#include <vector> #include <vector>
@ -42,8 +43,6 @@
namespace rocksdb { namespace rocksdb {
using std::dynamic_pointer_cast;
SstFileReader::SstFileReader(const std::string& file_path, SstFileReader::SstFileReader(const std::string& file_path,
bool verify_checksum, bool verify_checksum,
bool output_hex) bool output_hex)
@ -115,18 +114,13 @@ Status SstFileReader::NewTableReader(
unique_ptr<TableReader>* table_reader) { unique_ptr<TableReader>* table_reader) {
// We need to turn off pre-fetching of index and filter nodes for // We need to turn off pre-fetching of index and filter nodes for
// BlockBasedTable // BlockBasedTable
shared_ptr<BlockBasedTableFactory> block_table_factory = if (BlockBasedTableFactory::kName == options_.table_factory->Name()) {
dynamic_pointer_cast<BlockBasedTableFactory>(options_.table_factory); return options_.table_factory->NewTableReader(
if (block_table_factory) {
return block_table_factory->NewTableReader(
TableReaderOptions(ioptions_, soptions_, internal_comparator_, TableReaderOptions(ioptions_, soptions_, internal_comparator_,
/*skip_filters=*/false), /*skip_filters=*/false),
std::move(file_), file_size, &table_reader_, /*enable_prefetch=*/false); std::move(file_), file_size, &table_reader_, /*enable_prefetch=*/false);
} }
assert(!block_table_factory);
// For all other factory implementation // For all other factory implementation
return options_.table_factory->NewTableReader( return options_.table_factory->NewTableReader(
TableReaderOptions(ioptions_, soptions_, internal_comparator_), TableReaderOptions(ioptions_, soptions_, internal_comparator_),

@ -0,0 +1,19 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
namespace rocksdb {
// The helper function to assert the move from dynamic_cast<> to
// static_cast<> is correct. This function is to deal with legacy code.
// It is not recommanded to add new code to issue class casting. The preferred
// solution is to implement the functionality without a need of casting.
template <class DestClass, class SrcClass>
inline DestClass* static_cast_with_check(SrcClass* x) {
DestClass* ret = static_cast<DestClass*>(x);
#ifdef ROCKSDB_USE_RTTI
assert(ret == dynamic_cast<DestClass*>(x));
#endif
return ret;
}
} // namespace rocksdb

@ -23,6 +23,7 @@
#include "table/block_based_table_builder.h" #include "table/block_based_table_builder.h"
#include "table/block_builder.h" #include "table/block_builder.h"
#include "table/meta_blocks.h" #include "table/meta_blocks.h"
#include "util/cast_util.h"
#include "util/crc32c.h" #include "util/crc32c.h"
#include "util/file_reader_writer.h" #include "util/file_reader_writer.h"
#include "util/filename.h" #include "util/filename.h"
@ -199,7 +200,8 @@ BlobDBImpl::BlobDBImpl(const std::string& dbname,
open_p1_done_(false), open_p1_done_(false),
debug_level_(0) { debug_level_(0) {
const BlobDBOptionsImpl* options_impl = const BlobDBOptionsImpl* options_impl =
dynamic_cast<const BlobDBOptionsImpl*>(&blob_db_options); static_cast_with_check<const BlobDBOptionsImpl, const BlobDBOptions>(
&blob_db_options);
if (options_impl) { if (options_impl) {
bdb_options_ = *options_impl; bdb_options_ = *options_impl;
} }
@ -215,12 +217,7 @@ Status BlobDBImpl::LinkToBaseDB(DB* db) {
db_ = db; db_ = db;
// the Base DB in-itself can be a stackable DB // the Base DB in-itself can be a stackable DB
StackableDB* sdb = dynamic_cast<StackableDB*>(db_); db_impl_ = static_cast_with_check<DBImpl, DB>(db_->GetRootDB());
if (sdb) {
db_impl_ = dynamic_cast<DBImpl*>(sdb->GetBaseDB());
} else {
db_impl_ = dynamic_cast<DBImpl*>(db);
}
env_ = db_->GetEnv(); env_ = db_->GetEnv();
@ -249,7 +246,7 @@ BlobDBOptions BlobDBImpl::GetBlobDBOptions() const { return bdb_options_; }
BlobDBImpl::BlobDBImpl(DB* db, const BlobDBOptions& blob_db_options) BlobDBImpl::BlobDBImpl(DB* db, const BlobDBOptions& blob_db_options)
: BlobDB(db), : BlobDB(db),
db_impl_(dynamic_cast<DBImpl*>(db)), db_impl_(static_cast_with_check<DBImpl, DB>(db)),
opt_db_(new OptimisticTransactionDBImpl(db, false)), opt_db_(new OptimisticTransactionDBImpl(db, false)),
wo_set_(false), wo_set_(false),
bdb_options_(blob_db_options), bdb_options_(blob_db_options),
@ -268,10 +265,9 @@ BlobDBImpl::BlobDBImpl(DB* db, const BlobDBOptions& blob_db_options)
total_blob_space_(0) { total_blob_space_(0) {
assert(db_impl_ != nullptr); assert(db_impl_ != nullptr);
const BlobDBOptionsImpl* options_impl = const BlobDBOptionsImpl* options_impl =
dynamic_cast<const BlobDBOptionsImpl*>(&blob_db_options); static_cast_with_check<const BlobDBOptionsImpl, const BlobDBOptions>(
if (options_impl) { &blob_db_options);
bdb_options_ = *options_impl; bdb_options_ = *options_impl;
}
if (!bdb_options_.blob_dir.empty()) if (!bdb_options_.blob_dir.empty())
blob_dir_ = (bdb_options_.path_relative) blob_dir_ = (bdb_options_.path_relative)
@ -1752,8 +1748,8 @@ Status BlobDBImpl::GCFileAndUpdateLSM(const std::shared_ptr<BlobFile>& bfptr,
gcstats->deleted_size += record.GetBlobSize(); gcstats->deleted_size += record.GetBlobSize();
if (first_gc) continue; if (first_gc) continue;
Transaction* txn = static_cast<OptimisticTransactionDB*>(opt_db_.get()) Transaction* txn = opt_db_->BeginTransaction(
->BeginTransaction(write_options_); write_options_, OptimisticTransactionOptions(), nullptr);
txn->Delete(cfh, record.Key()); txn->Delete(cfh, record.Key());
Status s1 = txn->Commit(); Status s1 = txn->Commit();
// chances that this DELETE will fail is low. If it fails, it would be // chances that this DELETE will fail is low. If it fails, it would be
@ -1817,8 +1813,8 @@ Status BlobDBImpl::GCFileAndUpdateLSM(const std::shared_ptr<BlobFile>& bfptr,
newfile->file_size_ += BlobLogRecord::kHeaderSize + record.Key().size() + newfile->file_size_ += BlobLogRecord::kHeaderSize + record.Key().size() +
record.Blob().size() + BlobLogRecord::kFooterSize; record.Blob().size() + BlobLogRecord::kFooterSize;
Transaction* txn = static_cast<OptimisticTransactionDB*>(opt_db_.get()) Transaction* txn = opt_db_->BeginTransaction(
->BeginTransaction(write_options_); write_options_, OptimisticTransactionOptions(), nullptr);
txn->Put(cfh, record.Key(), index_entry); txn->Put(cfh, record.Key(), index_entry);
Status s1 = txn->Commit(); Status s1 = txn->Commit();
// chances that this Put will fail is low. If it fails, it would be because // chances that this Put will fail is low. If it fails, it would be because

@ -51,11 +51,9 @@ void ColumnAwareEncodingReader::InitTableReader(const std::string& file_path) {
options_.comparator = &internal_comparator_; options_.comparator = &internal_comparator_;
options_.table_factory = std::make_shared<BlockBasedTableFactory>(); options_.table_factory = std::make_shared<BlockBasedTableFactory>();
shared_ptr<BlockBasedTableFactory> block_table_factory =
std::dynamic_pointer_cast<BlockBasedTableFactory>(options_.table_factory);
std::unique_ptr<TableReader> table_reader; std::unique_ptr<TableReader> table_reader;
block_table_factory->NewTableReader( options_.table_factory->NewTableReader(
TableReaderOptions(ioptions_, soptions_, internal_comparator_, TableReaderOptions(ioptions_, soptions_, internal_comparator_,
/*skip_filters=*/false), /*skip_filters=*/false),
std::move(file_), file_size, &table_reader, /*enable_prefetch=*/false); std::move(file_), file_size, &table_reader, /*enable_prefetch=*/false);

@ -100,28 +100,34 @@ class DummyTableFactory : public TableFactory {
DummyTableFactory() {} DummyTableFactory() {}
virtual ~DummyTableFactory() {} virtual ~DummyTableFactory() {}
virtual const char* Name() const { return "DummyTableFactory"; } virtual const char* Name() const override { return "DummyTableFactory"; }
virtual Status NewTableReader(const TableReaderOptions& table_reader_options, virtual Status NewTableReader(
unique_ptr<RandomAccessFileReader>&& file, const TableReaderOptions& table_reader_options,
uint64_t file_size, unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
unique_ptr<TableReader>* table_reader, unique_ptr<TableReader>* table_reader,
bool prefetch_index_and_filter_in_cache) const { bool prefetch_index_and_filter_in_cache) const override {
return Status::NotSupported(); return Status::NotSupported();
} }
virtual TableBuilder* NewTableBuilder( virtual TableBuilder* NewTableBuilder(
const TableBuilderOptions& table_builder_options, const TableBuilderOptions& table_builder_options,
uint32_t column_family_id, WritableFileWriter* file) const { uint32_t column_family_id, WritableFileWriter* file) const override {
return nullptr; return nullptr;
} }
virtual Status SanitizeOptions(const DBOptions& db_opts, virtual Status SanitizeOptions(
const ColumnFamilyOptions& cf_opts) const { const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts) const override {
return Status::NotSupported(); return Status::NotSupported();
} }
virtual std::string GetPrintableTableOptions() const { return ""; } virtual std::string GetPrintableTableOptions() const override { return ""; }
Status GetOptionString(std::string* opt_string,
const std::string& delimiter) const override {
return Status::OK();
}
}; };
class DummyMergeOperator : public MergeOperator { class DummyMergeOperator : public MergeOperator {

@ -17,6 +17,7 @@
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/utilities/optimistic_transaction_db.h"
#include "util/cast_util.h"
#include "util/string_util.h" #include "util/string_util.h"
#include "utilities/transactions/transaction_util.h" #include "utilities/transactions/transaction_util.h"
@ -62,13 +63,7 @@ Status OptimisticTransactionImpl::Commit() {
// check whether this transaction is safe to be committed. // check whether this transaction is safe to be committed.
OptimisticTransactionCallback callback(this); OptimisticTransactionCallback callback(this);
DBImpl* db_impl = dynamic_cast<DBImpl*>(db_->GetRootDB()); DBImpl* db_impl = static_cast_with_check<DBImpl, DB>(db_->GetRootDB());
if (db_impl == nullptr) {
// This should only happen if we support creating transactions from
// a StackableDB and someone overrides GetRootDB().
return Status::InvalidArgument(
"DB::GetRootDB() returned an unexpected DB class");
}
Status s = db_impl->WriteWithCallback( Status s = db_impl->WriteWithCallback(
write_options_, GetWriteBatch()->GetWriteBatch(), &callback); write_options_, GetWriteBatch()->GetWriteBatch(), &callback);
@ -122,8 +117,7 @@ Status OptimisticTransactionImpl::TryLock(ColumnFamilyHandle* column_family,
Status OptimisticTransactionImpl::CheckTransactionForConflicts(DB* db) { Status OptimisticTransactionImpl::CheckTransactionForConflicts(DB* db) {
Status result; Status result;
assert(dynamic_cast<DBImpl*>(db) != nullptr); auto db_impl = static_cast_with_check<DBImpl, DB>(db);
auto db_impl = reinterpret_cast<DBImpl*>(db);
// Since we are on the write thread and do not want to block other writers, // Since we are on the write thread and do not want to block other writers,
// we will do a cache-only conflict check. This can result in TryAgain // we will do a cache-only conflict check. This can result in TryAgain

@ -15,6 +15,7 @@
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "rocksdb/options.h" #include "rocksdb/options.h"
#include "rocksdb/utilities/transaction_db.h" #include "rocksdb/utilities/transaction_db.h"
#include "util/cast_util.h"
#include "utilities/transactions/transaction_db_mutex_impl.h" #include "utilities/transactions/transaction_db_mutex_impl.h"
#include "utilities/transactions/transaction_impl.h" #include "utilities/transactions/transaction_impl.h"
@ -23,7 +24,7 @@ namespace rocksdb {
TransactionDBImpl::TransactionDBImpl(DB* db, TransactionDBImpl::TransactionDBImpl(DB* db,
const TransactionDBOptions& txn_db_options) const TransactionDBOptions& txn_db_options)
: TransactionDB(db), : TransactionDB(db),
db_impl_(dynamic_cast<DBImpl*>(db)), db_impl_(static_cast_with_check<DBImpl, DB>(db)),
txn_db_options_(txn_db_options), txn_db_options_(txn_db_options),
lock_mgr_(this, txn_db_options_.num_stripes, txn_db_options.max_num_locks, lock_mgr_(this, txn_db_options_.num_stripes, txn_db_options.max_num_locks,
txn_db_options_.custom_mutex_factory txn_db_options_.custom_mutex_factory
@ -52,7 +53,7 @@ TransactionDBImpl::TransactionDBImpl(DB* db,
TransactionDBImpl::TransactionDBImpl(StackableDB* db, TransactionDBImpl::TransactionDBImpl(StackableDB* db,
const TransactionDBOptions& txn_db_options) const TransactionDBOptions& txn_db_options)
: TransactionDB(db), : TransactionDB(db),
db_impl_(dynamic_cast<DBImpl*>(db->GetRootDB())), db_impl_(static_cast_with_check<DBImpl, DB>(db->GetRootDB())),
txn_db_options_(txn_db_options), txn_db_options_(txn_db_options),
lock_mgr_(this, txn_db_options_.num_stripes, txn_db_options.max_num_locks, lock_mgr_(this, txn_db_options_.num_stripes, txn_db_options.max_num_locks,
txn_db_options_.custom_mutex_factory txn_db_options_.custom_mutex_factory
@ -371,8 +372,7 @@ Status TransactionDBImpl::Write(const WriteOptions& opts, WriteBatch* updates) {
Transaction* txn = BeginInternalTransaction(opts); Transaction* txn = BeginInternalTransaction(opts);
txn->DisableIndexing(); txn->DisableIndexing();
assert(dynamic_cast<TransactionImpl*>(txn) != nullptr); auto txn_impl = static_cast_with_check<TransactionImpl, Transaction>(txn);
auto txn_impl = reinterpret_cast<TransactionImpl*>(txn);
// Since commitBatch sorts the keys before locking, concurrent Write() // Since commitBatch sorts the keys before locking, concurrent Write()
// operations will not cause a deadlock. // operations will not cause a deadlock.
@ -412,8 +412,7 @@ bool TransactionDBImpl::TryStealingExpiredTransactionLocks(
void TransactionDBImpl::ReinitializeTransaction( void TransactionDBImpl::ReinitializeTransaction(
Transaction* txn, const WriteOptions& write_options, Transaction* txn, const WriteOptions& write_options,
const TransactionOptions& txn_options) { const TransactionOptions& txn_options) {
assert(dynamic_cast<TransactionImpl*>(txn) != nullptr); auto txn_impl = static_cast_with_check<TransactionImpl, Transaction>(txn);
auto txn_impl = reinterpret_cast<TransactionImpl*>(txn);
txn_impl->Reinitialize(this, write_options, txn_options); txn_impl->Reinitialize(this, write_options, txn_options);
} }

@ -19,6 +19,7 @@
#include "rocksdb/snapshot.h" #include "rocksdb/snapshot.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/utilities/transaction_db.h" #include "rocksdb/utilities/transaction_db.h"
#include "util/cast_util.h"
#include "util/string_util.h" #include "util/string_util.h"
#include "util/sync_point.h" #include "util/sync_point.h"
#include "utilities/transactions/transaction_db_impl.h" #include "utilities/transactions/transaction_db_impl.h"
@ -46,10 +47,9 @@ TransactionImpl::TransactionImpl(TransactionDB* txn_db,
lock_timeout_(0), lock_timeout_(0),
deadlock_detect_(false), deadlock_detect_(false),
deadlock_detect_depth_(0) { deadlock_detect_depth_(0) {
txn_db_impl_ = dynamic_cast<TransactionDBImpl*>(txn_db); txn_db_impl_ =
assert(txn_db_impl_); static_cast_with_check<TransactionDBImpl, TransactionDB>(txn_db);
db_impl_ = dynamic_cast<DBImpl*>(txn_db->GetRootDB()); db_impl_ = static_cast_with_check<DBImpl, DB>(txn_db->GetRootDB());
assert(db_impl_);
Initialize(txn_options); Initialize(txn_options);
} }
@ -526,8 +526,7 @@ Status TransactionImpl::ValidateSnapshot(ColumnFamilyHandle* column_family,
*new_seqno = seq; *new_seqno = seq;
assert(dynamic_cast<DBImpl*>(db_) != nullptr); auto db_impl = static_cast_with_check<DBImpl, DB>(db_);
auto db_impl = reinterpret_cast<DBImpl*>(db_);
ColumnFamilyHandle* cfh = ColumnFamilyHandle* cfh =
column_family ? column_family : db_impl->DefaultColumnFamily(); column_family ? column_family : db_impl->DefaultColumnFamily();

@ -22,6 +22,7 @@
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "rocksdb/utilities/transaction_db_mutex.h" #include "rocksdb/utilities/transaction_db_mutex.h"
#include "util/cast_util.h"
#include "util/murmurhash.h" #include "util/murmurhash.h"
#include "util/sync_point.h" #include "util/sync_point.h"
#include "util/thread_local.h" #include "util/thread_local.h"
@ -112,8 +113,9 @@ TransactionLockMgr::TransactionLockMgr(
max_num_locks_(max_num_locks), max_num_locks_(max_num_locks),
lock_maps_cache_(new ThreadLocalPtr(&UnrefLockMapsCache)), lock_maps_cache_(new ThreadLocalPtr(&UnrefLockMapsCache)),
mutex_factory_(mutex_factory) { mutex_factory_(mutex_factory) {
txn_db_impl_ = dynamic_cast<TransactionDBImpl*>(txn_db); assert(txn_db);
assert(txn_db_impl_); txn_db_impl_ =
static_cast_with_check<TransactionDBImpl, TransactionDB>(txn_db);
} }
TransactionLockMgr::~TransactionLockMgr() {} TransactionLockMgr::~TransactionLockMgr() {}

Loading…
Cancel
Save