NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optimize SpatialDB

Browse Source 
Summary:
Two things:
1. Use hash-based index for data column family
2. Use Get() instead of Iterator Seek() when DB is opened read-only

Test Plan: added read-only test in unit test

Reviewers: yinwang

Reviewed By: yinwang

Subscribers: leveldb

Differential Revision: https://reviews.facebook.net/D22323
main
Igor Canadi 10 years ago
parent 23861857c4
commit ff6ec0eb17
2 changed files with 219 additions and 105 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 244
      utilities/spatialdb/spatial_db.cc
  2. 80
      utilities/spatialdb/spatial_db_test.cc

244
utilities/spatialdb/spatial_db.cc
Unescape View File

@ -16,6 +16,8 @@
#include <unordered_set>
#include "rocksdb/cache.h"
#include "rocksdb/options.h"
#include "rocksdb/slice_transform.h"
#include "rocksdb/table.h"
#include "rocksdb/db.h"
#include "rocksdb/utilities/stackable_db.h"
@ -244,13 +246,76 @@ std::string FeatureSet::DebugString() const {
return out + "}";
}
class ValueGetter {
public:
ValueGetter() {}
virtual ~ValueGetter() {}
virtual bool Get(uint64_t id) = 0;
virtual const Slice value() const = 0;
virtual Status status() const = 0;
};
class ValueGetterFromDB : public ValueGetter {
public:
ValueGetterFromDB(DB* db, ColumnFamilyHandle* cf) : db_(db), cf_(cf) {}
virtual bool Get(uint64_t id) override {
std::string encoded_id;
PutFixed64BigEndian(&encoded_id, id);
status_ = db_->Get(ReadOptions(), cf_, encoded_id, &value_);
if (status_.IsNotFound()) {
status_ = Status::Corruption("Index inconsistency");
return false;
}
return true;
}
virtual const Slice value() const override { return value_; }
virtual Status status() const override { return status_; }
private:
std::string value_;
DB* db_;
ColumnFamilyHandle* cf_;
Status status_;
};
class ValueGetterFromIterator : public ValueGetter {
public:
explicit ValueGetterFromIterator(Iterator* iterator) : iterator_(iterator) {}
virtual bool Get(uint64_t id) override {
std::string encoded_id;
PutFixed64BigEndian(&encoded_id, id);
iterator_->Seek(encoded_id);
if (!iterator_->Valid() || iterator_->key() != Slice(encoded_id)) {
status_ = Status::Corruption("Index inconsistency");
return false;
}
return true;
}
virtual const Slice value() const override { return iterator_->value(); }
virtual Status status() const override { return status_; }
private:
std::unique_ptr<Iterator> iterator_;
Status status_;
};
class SpatialIndexCursor : public Cursor {
public:
// tile_box is inclusive
SpatialIndexCursor(Iterator* spatial_iterator, Iterator* data_iterator,
SpatialIndexCursor(Iterator* spatial_iterator, ValueGetter* value_getter,
const BoundingBox<uint64_t>& tile_bbox, uint32_t tile_bits)
: data_iterator_(data_iterator),
valid_(true) {
: value_getter_(value_getter), valid_(true) {
// calculate quad keys we'll need to query
std::vector<uint64_t> quad_keys;
quad_keys.reserve((tile_bbox.max_x - tile_bbox.min_x + 1) *
@ -329,7 +394,7 @@ class SpatialIndexCursor : public Cursor {
if (!status_.ok()) {
return status_;
}
return data_iterator_->status();
return value_getter_->status();
}
private:
@ -356,32 +421,23 @@ class SpatialIndexCursor : public Cursor {
return true;
}
// doesn't return anything, but sets valid_ and status_ on corruption
void ExtractData() {
assert(valid_);
std::string encoded_id;
PutFixed64BigEndian(&encoded_id, *primary_keys_iterator_);
valid_ = value_getter_->Get(*primary_keys_iterator_);
data_iterator_->Seek(encoded_id);
if (!data_iterator_->Valid() ||
data_iterator_->key() != Slice(encoded_id)) {
status_ = Status::Corruption("Index inconsistency");
valid_ = false;
return;
if (valid_) {
Slice data = value_getter_->value();
current_feature_set_.Clear();
if (!GetLengthPrefixedSlice(&data, &current_blob_) ||
!current_feature_set_.Deserialize(data)) {
status_ = Status::Corruption("Primary key column family corruption");
valid_ = false;
}
}
Slice data = data_iterator_->value();
current_feature_set_.Clear();
if (!GetLengthPrefixedSlice(&data, &current_blob_) ||
!current_feature_set_.Deserialize(data)) {
status_ = Status::Corruption("Primary key column family corruption");
valid_ = false;
return;
}
}
unique_ptr<Iterator> data_iterator_;
unique_ptr<ValueGetter> value_getter_;
bool valid_;
Status status_;
@ -427,10 +483,11 @@ class SpatialDBImpl : public SpatialDB {
DB* db, ColumnFamilyHandle* data_column_family,
const std::vector<std::pair<SpatialIndexOptions, ColumnFamilyHandle*>>&
spatial_indexes,
uint64_t next_id)
uint64_t next_id, bool read_only)
: SpatialDB(db),
data_column_family_(data_column_family),
next_id_(next_id) {
next_id_(next_id),
read_only_(read_only) {
for (const auto& index : spatial_indexes) {
name_to_index_.insert(
{index.first.name, IndexColumnFamily(index.first, index.second)});
@ -521,17 +578,26 @@ class SpatialDBImpl : public SpatialDB {
return new ErrorCursor(Status::InvalidArgument(
"Spatial index " + spatial_index + " not found"));
}
const auto& si = itr->second.index;
Iterator* spatial_iterator;
ValueGetter* value_getter;
std::vector<Iterator*> iterators;
Status s = NewIterators(read_options,
{data_column_family_, itr->second.column_family},
&iterators);
if (!s.ok()) {
return new ErrorCursor(s);
}
if (read_only_) {
spatial_iterator = NewIterator(read_options, itr->second.column_family);
value_getter = new ValueGetterFromDB(this, data_column_family_);
} else {
std::vector<Iterator*> iterators;
Status s = NewIterators(read_options,
{data_column_family_, itr->second.column_family},
&iterators);
if (!s.ok()) {
return new ErrorCursor(s);
}
const auto& si = itr->second.index;
return new SpatialIndexCursor(iterators[1], iterators[0],
spatial_iterator = iterators[1];
value_getter = new ValueGetterFromIterator(iterators[0]);
}
return new SpatialIndexCursor(spatial_iterator, value_getter,
GetTileBoundingBox(si, bbox), si.tile_bits);
}
@ -548,31 +614,61 @@ class SpatialDBImpl : public SpatialDB {
std::unordered_map<std::string, IndexColumnFamily> name_to_index_;
std::atomic<uint64_t> next_id_;
bool read_only_;
};
namespace {
Options GetRocksDBOptionsFromOptions(const SpatialDBOptions& options) {
Options rocksdb_options;
rocksdb_options.IncreaseParallelism(options.num_threads);
rocksdb_options.write_buffer_size = 256 * 1024 * 1024; // 256MB
rocksdb_options.max_bytes_for_level_base = 1024 * 1024 * 1024; // 1 GB
DBOptions GetDBOptions(const SpatialDBOptions& options) {
DBOptions db_options;
db_options.IncreaseParallelism(options.num_threads);
if (options.bulk_load) {
db_options.disableDataSync = true;
}
return db_options;
}
ColumnFamilyOptions GetColumnFamilyOptions(const SpatialDBOptions& options,
std::shared_ptr<Cache> block_cache) {
ColumnFamilyOptions column_family_options;
column_family_options.write_buffer_size = 256 * 1024 * 1024; // 256MB
column_family_options.max_bytes_for_level_base = 1024 * 1024 * 1024; // 1 GB
// only compress levels >= 1
rocksdb_options.compression_per_level.resize(rocksdb_options.num_levels);
for (int i = 0; i < rocksdb_options.num_levels; ++i) {
column_family_options.compression_per_level.resize(
column_family_options.num_levels);
for (int i = 0; i < column_family_options.num_levels; ++i) {
if (i == 0) {
rocksdb_options.compression_per_level[i] = kNoCompression;
column_family_options.compression_per_level[i] = kNoCompression;
} else {
rocksdb_options.compression_per_level[i] = kLZ4Compression;
column_family_options.compression_per_level[i] = kLZ4Compression;
}
}
BlockBasedTableOptions table_options;
table_options.block_cache = NewLRUCache(options.cache_size);
rocksdb_options.table_factory.reset(NewBlockBasedTableFactory(table_options));
table_options.block_cache = block_cache;
column_family_options.table_factory.reset(
NewBlockBasedTableFactory(table_options));
if (options.bulk_load) {
rocksdb_options.PrepareForBulkLoad();
}
return rocksdb_options;
column_family_options.level0_file_num_compaction_trigger = (1 << 30);
column_family_options.level0_slowdown_writes_trigger = (1 << 30);
column_family_options.level0_stop_writes_trigger = (1 << 30);
column_family_options.disable_auto_compactions = true;
column_family_options.source_compaction_factor = (1 << 30);
column_family_options.num_levels = 2;
column_family_options.target_file_size_base = 256 * 1024 * 1024;
column_family_options.max_mem_compaction_level = 0;
}
return column_family_options;
}
ColumnFamilyOptions OptimizeOptionsForDataColumnFamily(
ColumnFamilyOptions options, std::shared_ptr<Cache> block_cache) {
options.prefix_extractor.reset(NewNoopTransform());
BlockBasedTableOptions block_based_options;
block_based_options.index_type = BlockBasedTableOptions::kHashSearch;
block_based_options.block_cache = block_cache;
options.table_factory.reset(NewBlockBasedTableFactory(block_based_options));
return options;
}
} // namespace
class MetadataStorage {
@ -618,26 +714,30 @@ class MetadataStorage {
Status SpatialDB::Create(
const SpatialDBOptions& options, const std::string& name,
const std::vector<SpatialIndexOptions>& spatial_indexes) {
Options rocksdb_options = GetRocksDBOptionsFromOptions(options);
rocksdb_options.create_if_missing = true;
rocksdb_options.create_missing_column_families = true;
rocksdb_options.error_if_exists = true;
DBOptions db_options = GetDBOptions(options);
db_options.create_if_missing = true;
db_options.create_missing_column_families = true;
db_options.error_if_exists = true;
auto block_cache = NewLRUCache(options.cache_size);
ColumnFamilyOptions column_family_options =
GetColumnFamilyOptions(options, block_cache);
std::vector<ColumnFamilyDescriptor> column_families;
column_families.push_back(ColumnFamilyDescriptor(
kDefaultColumnFamilyName, ColumnFamilyOptions(rocksdb_options)));
column_families.push_back(ColumnFamilyDescriptor(
kMetadataColumnFamilyName, ColumnFamilyOptions(rocksdb_options)));
kDefaultColumnFamilyName,
OptimizeOptionsForDataColumnFamily(column_family_options, block_cache)));
column_families.push_back(
ColumnFamilyDescriptor(kMetadataColumnFamilyName, column_family_options));
for (const auto& index : spatial_indexes) {
column_families.emplace_back(GetSpatialIndexColumnFamilyName(index.name),
ColumnFamilyOptions(rocksdb_options));
column_family_options);
}
std::vector<ColumnFamilyHandle*> handles;
DB* base_db;
Status s = DB::Open(DBOptions(rocksdb_options), name, column_families,
&handles, &base_db);
Status s = DB::Open(db_options, name, column_families, &handles, &base_db);
if (!s.ok()) {
return s;
}
@ -659,13 +759,15 @@ Status SpatialDB::Create(
Status SpatialDB::Open(const SpatialDBOptions& options, const std::string& name,
SpatialDB** db, bool read_only) {
Options rocksdb_options = GetRocksDBOptionsFromOptions(options);
DBOptions db_options = GetDBOptions(options);
auto block_cache = NewLRUCache(options.cache_size);
ColumnFamilyOptions column_family_options =
GetColumnFamilyOptions(options, block_cache);
Status s;
std::vector<std::string> existing_column_families;
std::vector<std::string> spatial_indexes;
s = DB::ListColumnFamilies(DBOptions(rocksdb_options), name,
&existing_column_families);
s = DB::ListColumnFamilies(db_options, name, &existing_column_families);
if (!s.ok()) {
return s;
}
@ -678,22 +780,22 @@ Status SpatialDB::Open(const SpatialDBOptions& options, const std::string& name,
std::vector<ColumnFamilyDescriptor> column_families;
column_families.push_back(ColumnFamilyDescriptor(
kDefaultColumnFamilyName, ColumnFamilyOptions(rocksdb_options)));
column_families.push_back(ColumnFamilyDescriptor(
kMetadataColumnFamilyName, ColumnFamilyOptions(rocksdb_options)));
kDefaultColumnFamilyName,
OptimizeOptionsForDataColumnFamily(column_family_options, block_cache)));
column_families.push_back(
ColumnFamilyDescriptor(kMetadataColumnFamilyName, column_family_options));
for (const auto& index : spatial_indexes) {
column_families.emplace_back(GetSpatialIndexColumnFamilyName(index),
ColumnFamilyOptions(rocksdb_options));
column_family_options);
}
std::vector<ColumnFamilyHandle*> handles;
DB* base_db;
if (read_only) {
s = DB::OpenForReadOnly(DBOptions(rocksdb_options), name, column_families,
&handles, &base_db);
s = DB::OpenForReadOnly(db_options, name, column_families, &handles,
&base_db);
} else {
s = DB::Open(DBOptions(rocksdb_options), name, column_families, &handles,
&base_db);
s = DB::Open(db_options, name, column_families, &handles, &base_db);
}
if (!s.ok()) {
return s;
@ -730,13 +832,13 @@ Status SpatialDB::Open(const SpatialDBOptions& options, const std::string& name,
for (auto h : handles) {
delete h;
}
delete db;
delete base_db;
return s;
}
// I don't need metadata column family any more, so delete it
delete handles[1];
*db = new SpatialDBImpl(base_db, handles[0], index_cf, next_id);
*db = new SpatialDBImpl(base_db, handles[0], index_cf, next_id, read_only);
return Status::OK();
}

80
utilities/spatialdb/spatial_db_test.cc
Unescape View File

@ -151,41 +151,53 @@ TEST(SpatialDBTest, FeatureSetTest) {
}
TEST(SpatialDBTest, SimpleTest) {
ASSERT_OK(SpatialDB::Create(
SpatialDBOptions(), dbname_,
{SpatialIndexOptions("index", BoundingBox<double>(0, 0, 128, 128), 3)}));
ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(33, 17, 63, 79),
"one", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(65, 65, 111, 111),
"two", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(1, 49, 127, 63),
"three", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(20, 100, 21, 101),
"four", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(81, 33, 127, 63),
"five", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(1, 65, 47, 95),
"six", FeatureSet(), {"index"}));
AssertCursorResults(BoundingBox<double>(33, 17, 47, 31), "index", {"one"});
AssertCursorResults(BoundingBox<double>(17, 33, 79, 63), "index",
{"one", "three"});
AssertCursorResults(BoundingBox<double>(17, 81, 63, 111), "index",
{"four", "six"});
AssertCursorResults(BoundingBox<double>(85, 86, 85, 86), "index", {"two"});
AssertCursorResults(BoundingBox<double>(33, 1, 127, 111), "index",
{"one", "two", "three", "five", "six"});
// even though the bounding box doesn't intersect, we got "four" back because
// it's in the same tile
AssertCursorResults(BoundingBox<double>(18, 98, 19, 99), "index", {"four"});
AssertCursorResults(BoundingBox<double>(130, 130, 131, 131), "index", {});
AssertCursorResults(BoundingBox<double>(81, 17, 127, 31), "index", {});
AssertCursorResults(BoundingBox<double>(90, 50, 91, 51), "index",
{"three", "five"});
// iter 0 -- not read only
// iter 1 -- read only
for (int iter = 0; iter < 2; ++iter) {
DestroyDB(dbname_, Options());
ASSERT_OK(SpatialDB::Create(
SpatialDBOptions(), dbname_,
{SpatialIndexOptions("index", BoundingBox<double>(0, 0, 128, 128),
3)}));
ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(33, 17, 63, 79),
"one", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(65, 65, 111, 111),
"two", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(1, 49, 127, 63),
"three", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(20, 100, 21, 101),
"four", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(81, 33, 127, 63),
"five", FeatureSet(), {"index"}));
ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(1, 65, 47, 95),
"six", FeatureSet(), {"index"}));
if (iter == 1) {
delete db_;
ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_, true));
}
delete db_;
AssertCursorResults(BoundingBox<double>(33, 17, 47, 31), "index", {"one"});
AssertCursorResults(BoundingBox<double>(17, 33, 79, 63), "index",
{"one", "three"});
AssertCursorResults(BoundingBox<double>(17, 81, 63, 111), "index",
{"four", "six"});
AssertCursorResults(BoundingBox<double>(85, 86, 85, 86), "index", {"two"});
AssertCursorResults(BoundingBox<double>(33, 1, 127, 111), "index",
{"one", "two", "three", "five", "six"});
// even though the bounding box doesn't intersect, we got "four" back
// because
// it's in the same tile
AssertCursorResults(BoundingBox<double>(18, 98, 19, 99), "index", {"four"});
AssertCursorResults(BoundingBox<double>(130, 130, 131, 131), "index", {});
AssertCursorResults(BoundingBox<double>(81, 17, 127, 31), "index", {});
AssertCursorResults(BoundingBox<double>(90, 50, 91, 51), "index",
{"three", "five"});
delete db_;
}
}
namespace {

Write Preview
Loading…
Cancel
Save