NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Unified maps with Comparator for sorting, other cleanup

Browse Source 
Summary:
This diff is a collection of cleanups that were initially part of D43179.
Additionally it adds a unified way of defining key-value maps that use a
Comparator for sorting (this was previously implemented in four different
places).

Test Plan: make clean check all

Reviewers: rven, anthony, yhchiang, sdong, igor

Reviewed By: igor

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D45993
main
Andres Noetzli 9 years ago
parent 3e0a672c50
commit 3c9cef1eed
24 changed files with 289 additions and 324 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 4
      db/builder.cc
  2. 24
      db/compaction_job.h
  3. 123
      db/compaction_job_test.cc
  4. 19
      db/comparator_db_test.cc
  5. 27
      db/db_impl.cc
  6. 22
      db/db_impl.h
  7. 2
      db/db_test.cc
  8. 7
      db/flush_job.cc
  9. 4
      db/flush_job_test.cc
  10. 49
      db/internal_stats.cc
  11. 6
      db/log_writer.h
  12. 10
      db/merge_helper.cc
  13. 2
      db/table_properties_collector_test.cc
  14. 8
      db/write_batch.cc
  15. 2
      include/rocksdb/utilities/write_batch_with_index.h
  16. 2
      table/meta_blocks.cc
  17. 23
      table/meta_blocks.h
  18. 35
      table/mock_table.cc
  19. 58
      table/mock_table.h
  20. 120
      table/table_test.cc
  21. 9
      util/hash_cuckoo_rep.cc
  22. 48
      util/stl_wrappers.h
  23. 6
      utilities/write_batch_with_index/write_batch_with_index.cc
  24. 3
      utilities/write_batch_with_index/write_batch_with_index_internal.h

4
db/builder.cc
Unescape View File

@ -137,7 +137,7 @@ Status BuildTable(
// In-memory key corruption is not ok;
// TODO: find a clean way to treat in memory key corruption
// Ugly walkaround to avoid compiler error for release build
// Ugly workaround to avoid compiler error for release build
bool ok __attribute__((unused)) = true;
ok = ParseInternalKey(key, &ikey);
assert(ok);
@ -159,7 +159,7 @@ Status BuildTable(
// If there are no snapshots, then this kv affect visibility at tip.
// Otherwise, search though all existing snapshots to find
// the earlist snapshot that is affected by this kv.
// the earliest snapshot that is affected by this kv.
SequenceNumber prev_snapshot = 0; // 0 means no previous snapshot
SequenceNumber key_needs_to_exist_in_snapshot =
EarliestVisibleSnapshot(ikey.sequence, snapshots, &prev_snapshot);

24
db/compaction_job.h
Unescape View File

@ -10,35 +10,35 @@
#include <atomic>
#include <deque>
#include <functional>
#include <limits>
#include <set>
#include <string>
#include <utility>
#include <vector>
#include <string>
#include <functional>
#include "db/column_family.h"
#include "db/dbformat.h"
#include "db/flush_scheduler.h"
#include "db/internal_stats.h"
#include "db/job_context.h"
#include "db/log_writer.h"
#include "db/column_family.h"
#include "db/version_edit.h"
#include "db/memtable_list.h"
#include "db/version_edit.h"
#include "db/write_controller.h"
#include "db/write_thread.h"
#include "port/port.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/compaction_job_stats.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/compaction_job_stats.h"
#include "rocksdb/transaction_log.h"
#include "util/autovector.h"
#include "util/event_logger.h"
#include "util/scoped_arena_iterator.h"
#include "util/stop_watch.h"
#include "util/thread_local.h"
#include "util/scoped_arena_iterator.h"
#include "db/internal_stats.h"
#include "db/write_controller.h"
#include "db/flush_scheduler.h"
#include "db/write_thread.h"
#include "db/job_context.h"
namespace rocksdb {

123
db/compaction_job_test.cc
Unescape View File

@ -92,7 +92,7 @@ class CompactionJobTest : public testing::Test {
return InternalKey(user_key, seq_num, t).Encode().ToString();
}
void AddMockFile(const mock::MockFileContents& contents, int level = 0) {
void AddMockFile(const stl_wrappers::KVMap& contents, int level = 0) {
assert(contents.size() > 0);
bool first_key = true;
@ -143,8 +143,8 @@ class CompactionJobTest : public testing::Test {
}
// returns expected result after compaction
mock::MockFileContents CreateTwoFiles(bool gen_corrupted_keys) {
mock::MockFileContents expected_results;
stl_wrappers::KVMap CreateTwoFiles(bool gen_corrupted_keys) {
auto expected_results = mock::MakeMockFile();
const int kKeysPerFile = 10000;
const int kCorruptKeysPerFile = 200;
const int kMatchingKeys = kKeysPerFile / 2;
@ -155,7 +155,7 @@ class CompactionJobTest : public testing::Test {
};
for (int i = 0; i < 2; ++i) {
mock::MockFileContents contents;
auto contents = mock::MakeMockFile();
for (int k = 0; k < kKeysPerFile; ++k) {
auto key = ToString(i * kMatchingKeys + k);
auto value = ToString(i * kKeysPerFile + k);
@ -215,7 +215,7 @@ class CompactionJobTest : public testing::Test {
}
void RunCompaction(const std::vector<std::vector<FileMetaData*>>& input_files,
const mock::MockFileContents& expected_results) {
const stl_wrappers::KVMap& expected_results) {
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
size_t num_input_files = 0;
@ -303,21 +303,16 @@ TEST_F(CompactionJobTest, SimpleCorrupted) {
TEST_F(CompactionJobTest, SimpleDeletion) {
NewDB();
mock::MockFileContents file1 = {
{ KeyStr("c", 4U, kTypeDeletion), "" },
{ KeyStr("c", 3U, kTypeValue), "val" }
};
auto file1 = mock::MakeMockFile({{KeyStr("c", 4U, kTypeDeletion), ""},
{KeyStr("c", 3U, kTypeValue), "val"}});
AddMockFile(file1);
mock::MockFileContents file2 = {
{ KeyStr("b", 2U, kTypeValue), "val" },
{ KeyStr("b", 1U, kTypeValue), "val" }
};
auto file2 = mock::MakeMockFile({{KeyStr("b", 2U, kTypeValue), "val"},
{KeyStr("b", 1U, kTypeValue), "val"}});
AddMockFile(file2);
mock::MockFileContents expected_results = {
{ KeyStr("b", 0U, kTypeValue), "val" }
};
auto expected_results =
mock::MakeMockFile({{KeyStr("b", 0U, kTypeValue), "val"}});
SetLastSequence(4U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
@ -327,22 +322,19 @@ TEST_F(CompactionJobTest, SimpleDeletion) {
TEST_F(CompactionJobTest, SimpleOverwrite) {
NewDB();
mock::MockFileContents file1 = {
{ KeyStr("a", 3U, kTypeValue), "val2" },
{ KeyStr("b", 4U, kTypeValue), "val3" },
};
auto file1 = mock::MakeMockFile({
{KeyStr("a", 3U, kTypeValue), "val2"},
{KeyStr("b", 4U, kTypeValue), "val3"},
});
AddMockFile(file1);
mock::MockFileContents file2 = {
{ KeyStr("a", 1U, kTypeValue), "val" },
{ KeyStr("b", 2U, kTypeValue), "val" }
};
auto file2 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"},
{KeyStr("b", 2U, kTypeValue), "val"}});
AddMockFile(file2);
mock::MockFileContents expected_results = {
{ KeyStr("a", 0U, kTypeValue), "val2" },
{ KeyStr("b", 0U, kTypeValue), "val3" }
};
auto expected_results =
mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "val2"},
{KeyStr("b", 0U, kTypeValue), "val3"}});
SetLastSequence(4U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
@ -352,30 +344,25 @@ TEST_F(CompactionJobTest, SimpleOverwrite) {
TEST_F(CompactionJobTest, SimpleNonLastLevel) {
NewDB();
mock::MockFileContents file1 = {
{ KeyStr("a", 5U, kTypeValue), "val2" },
{ KeyStr("b", 6U, kTypeValue), "val3" },
};
auto file1 = mock::MakeMockFile({
{KeyStr("a", 5U, kTypeValue), "val2"},
{KeyStr("b", 6U, kTypeValue), "val3"},
});
AddMockFile(file1);
mock::MockFileContents file2 = {
{ KeyStr("a", 3U, kTypeValue), "val" },
{ KeyStr("b", 4U, kTypeValue), "val" }
};
auto file2 = mock::MakeMockFile({{KeyStr("a", 3U, kTypeValue), "val"},
{KeyStr("b", 4U, kTypeValue), "val"}});
AddMockFile(file2, 1);
mock::MockFileContents file3 = {
{ KeyStr("a", 1U, kTypeValue), "val" },
{ KeyStr("b", 2U, kTypeValue), "val" }
};
auto file3 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"},
{KeyStr("b", 2U, kTypeValue), "val"}});
AddMockFile(file3, 2);
// Because level 1 is not the last level, the sequence numbers of a and b
// cannot be set to 0
mock::MockFileContents expected_results = {
{ KeyStr("a", 5U, kTypeValue), "val2" },
{ KeyStr("b", 6U, kTypeValue), "val3" }
};
auto expected_results =
mock::MakeMockFile({{KeyStr("a", 5U, kTypeValue), "val2"},
{KeyStr("b", 6U, kTypeValue), "val3"}});
SetLastSequence(6U);
auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
@ -387,23 +374,20 @@ TEST_F(CompactionJobTest, SimpleMerge) {
auto merge_op = MergeOperators::CreateStringAppendOperator();
NewDB(merge_op);
mock::MockFileContents file1 = {
{ KeyStr("a", 5U, kTypeMerge), "5" },
{ KeyStr("a", 4U, kTypeMerge), "4" },
{ KeyStr("a", 3U, kTypeValue), "3" },
};
auto file1 = mock::MakeMockFile({
{KeyStr("a", 5U, kTypeMerge), "5"},
{KeyStr("a", 4U, kTypeMerge), "4"},
{KeyStr("a", 3U, kTypeValue), "3"},
});
AddMockFile(file1);
mock::MockFileContents file2 = {
{ KeyStr("b", 2U, kTypeMerge), "2" },
{ KeyStr("b", 1U, kTypeValue), "1" }
};
auto file2 = mock::MakeMockFile(
{{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeValue), "1"}});
AddMockFile(file2);
mock::MockFileContents expected_results = {
{ KeyStr("a", 0U, kTypeValue), "3,4,5" },
{ KeyStr("b", 0U, kTypeValue), "1,2" }
};
auto expected_results =
mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"},
{KeyStr("b", 0U, kTypeValue), "1,2"}});
SetLastSequence(5U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);
@ -414,24 +398,21 @@ TEST_F(CompactionJobTest, NonAssocMerge) {
auto merge_op = MergeOperators::CreateStringAppendTESTOperator();
NewDB(merge_op);
mock::MockFileContents file1 = {
{ KeyStr("a", 5U, kTypeMerge), "5" },
{ KeyStr("a", 4U, kTypeMerge), "4" },
{ KeyStr("a", 3U, kTypeMerge), "3" },
};
auto file1 = mock::MakeMockFile({
{KeyStr("a", 5U, kTypeMerge), "5"},
{KeyStr("a", 4U, kTypeMerge), "4"},
{KeyStr("a", 3U, kTypeMerge), "3"},
});
AddMockFile(file1);
mock::MockFileContents file2 = {
{ KeyStr("b", 2U, kTypeMerge), "2" },
{ KeyStr("b", 1U, kTypeMerge), "1" }
};
auto file2 = mock::MakeMockFile(
{{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeMerge), "1"}});
AddMockFile(file2);
mock::MockFileContents expected_results = {
{ KeyStr("a", 0U, kTypeValue), "3,4,5" },
{ KeyStr("b", 2U, kTypeMerge), "2" },
{ KeyStr("b", 1U, kTypeMerge), "1" }
};
auto expected_results =
mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"},
{KeyStr("b", 2U, kTypeMerge), "2"},
{KeyStr("b", 1U, kTypeMerge), "1"}});
SetLastSequence(5U);
auto files = cfd_->current()->storage_info()->LevelFiles(0);

19
db/comparator_db_test.cc
Unescape View File

@ -10,6 +10,7 @@
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "util/hash.h"
#include "util/stl_wrappers.h"
#include "util/string_util.h"
#include "util/testharness.h"
#include "util/testutil.h"
@ -22,18 +23,10 @@ namespace {
static const Comparator* comparator;
// A comparator for std::map, using comparator
struct MapComparator {
bool operator()(const std::string& a, const std::string& b) const {
return comparator->Compare(a, b) < 0;
}
};
typedef std::map<std::string, std::string, MapComparator> KVMap;
class KVIter : public Iterator {
public:
explicit KVIter(const KVMap* map) : map_(map), iter_(map_->end()) {}
explicit KVIter(const stl_wrappers::KVMap* map)
: map_(map), iter_(map_->end()) {}
virtual bool Valid() const override { return iter_ != map_->end(); }
virtual void SeekToFirst() override { iter_ = map_->begin(); }
virtual void SeekToLast() override {
@ -60,8 +53,8 @@ class KVIter : public Iterator {
virtual Status status() const override { return Status::OK(); }
private:
const KVMap* const map_;
KVMap::const_iterator iter_;
const stl_wrappers::KVMap* const map_;
stl_wrappers::KVMap::const_iterator iter_;
};
void AssertItersEqual(Iterator* iter1, Iterator* iter2) {
@ -77,7 +70,7 @@ void AssertItersEqual(Iterator* iter1, Iterator* iter2) {
void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
Random* rnd, int num_writes, int num_iter_ops,
int num_trigger_flush) {
KVMap map;
stl_wrappers::KVMap map((stl_wrappers::LessOfComparator(comparator)));
for (int i = 0; i < num_writes; i++) {
if (num_trigger_flush > 0 && i != 0 && i % num_trigger_flush == 0) {

27
db/db_impl.cc
Unescape View File

@ -14,25 +14,27 @@
#endif
#include <inttypes.h>
#include <stdint.h>
#include <algorithm>
#include <climits>
#include <cstdio>
#include <set>
#include <stdexcept>
#include <stdint.h>
#include <string>
#include <unordered_set>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
#include "db/builder.h"
#include "db/flush_job.h"
#include "db/compaction_job.h"
#include "db/db_iter.h"
#include "db/dbformat.h"
#include "db/event_helpers.h"
#include "db/filename.h"
#include "db/flush_job.h"
#include "db/forward_iterator.h"
#include "db/job_context.h"
#include "db/log_reader.h"
#include "db/log_writer.h"
@ -43,24 +45,23 @@
#include "db/merge_helper.h"
#include "db/table_cache.h"
#include "db/table_properties_collector.h"
#include "db/forward_iterator.h"
#include "db/transaction_log_impl.h"
#include "db/version_set.h"
#include "db/writebuffer.h"
#include "db/write_batch_internal.h"
#include "db/write_callback.h"
#include "db/writebuffer.h"
#include "port/likely.h"
#include "port/port.h"
#include "rocksdb/cache.h"
#include "port/likely.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/db.h"
#include "rocksdb/delete_scheduler.h"
#include "rocksdb/env.h"
#include "rocksdb/merge_operator.h"
#include "rocksdb/version.h"
#include "rocksdb/statistics.h"
#include "rocksdb/status.h"
#include "rocksdb/table.h"
#include "rocksdb/version.h"
#include "table/block.h"
#include "table/block_based_table_factory.h"
#include "table/merger.h"
@ -75,16 +76,16 @@
#include "util/db_info_dumper.h"
#include "util/file_reader_writer.h"
#include "util/file_util.h"
#include "util/hash_skiplist_rep.h"
#include "util/hash_linklist_rep.h"
#include "util/logging.h"
#include "util/hash_skiplist_rep.h"
#include "util/iostats_context_imp.h"
#include "util/log_buffer.h"
#include "util/logging.h"
#include "util/mutexlock.h"
#include "util/perf_context_imp.h"
#include "util/iostats_context_imp.h"
#include "util/stop_watch.h"
#include "util/sync_point.h"
#include "util/string_util.h"
#include "util/sync_point.h"
#include "util/thread_status_updater.h"
#include "util/thread_status_util.h"
#include "util/xfunc.h"
@ -3274,8 +3275,8 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
// | | |
// +-------+-----------------------+
//
// ArenaWrappedDBIter inlines an arena area where all the iterartor in the
// the iterator tree is allocated in the order of being accessed when
// ArenaWrappedDBIter inlines an arena area where all the iterators in
// the iterator tree are allocated in the order of being accessed when
// querying.
// Laying out the iterators in the order of being accessed makes it more
// likely that any iterator pointer is close to the iterator it points to so

22
db/db_impl.h
Unescape View File

@ -11,21 +11,25 @@
#include <atomic>
#include <deque>
#include <limits>
#include <set>
#include <list>
#include <utility>
#include <list>
#include <vector>
#include <set>
#include <string>
#include <utility>
#include <vector>
#include "db/dbformat.h"
#include "db/column_family.h"
#include "db/compaction_job.h"
#include "db/dbformat.h"
#include "db/flush_job.h"
#include "db/flush_scheduler.h"
#include "db/internal_stats.h"
#include "db/log_writer.h"
#include "db/snapshot_impl.h"
#include "db/version_edit.h"
#include "db/wal_manager.h"
#include "db/write_controller.h"
#include "db/write_thread.h"
#include "db/writebuffer.h"
#include "memtable_list.h"
#include "port/port.h"
@ -36,15 +40,11 @@
#include "util/autovector.h"
#include "util/event_logger.h"
#include "util/hash.h"
#include "util/stop_watch.h"
#include "util/thread_local.h"
#include "util/scoped_arena_iterator.h"
#include "util/hash.h"
#include "util/instrumented_mutex.h"
#include "db/internal_stats.h"
#include "db/write_controller.h"
#include "db/flush_scheduler.h"
#include "db/write_thread.h"
#include "util/scoped_arena_iterator.h"
#include "util/stop_watch.h"
#include "util/thread_local.h"
namespace rocksdb {

2
db/db_test.cc
Unescape View File

@ -82,6 +82,7 @@ static long TestGetTickerCount(const Options& options, Tickers ticker_type) {
// `GetPropertiesOfAllTablesTest` is correct.
// This test assumes entries size is different for each of the tables.
namespace {
void VerifyTableProperties(DB* db, uint64_t expected_entries_size) {
TablePropertiesCollection props;
ASSERT_OK(db->GetPropertiesOfAllTables(&props));
@ -110,6 +111,7 @@ uint64_t GetNumberOfSstFilesForColumnFamily(DB* db,
}
return result;
}
} // namespace
class DBTest : public DBTestBase {

7
db/flush_job.cc
Unescape View File

@ -14,6 +14,7 @@
#endif
#include <inttypes.h>
#include <algorithm>
#include <vector>
@ -28,8 +29,8 @@
#include "db/memtable_list.h"
#include "db/merge_context.h"
#include "db/version_set.h"
#include "port/port.h"
#include "port/likely.h"
#include "port/port.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/statistics.h"
@ -43,11 +44,11 @@
#include "util/coding.h"
#include "util/event_logger.h"
#include "util/file_util.h"
#include "util/logging.h"
#include "util/iostats_context_imp.h"
#include "util/log_buffer.h"
#include "util/logging.h"
#include "util/mutexlock.h"
#include "util/perf_context_imp.h"
#include "util/iostats_context_imp.h"
#include "util/stop_watch.h"
#include "util/sync_point.h"
#include "util/thread_status_util.h"

4
db/flush_job_test.cc
Unescape View File

@ -104,7 +104,7 @@ TEST_F(FlushJobTest, NonEmpty) {
auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
kMaxSequenceNumber);
new_mem->Ref();
mock::MockFileContents inserted_keys;
auto inserted_keys = mock::MakeMockFile();
// Test data:
// seqno [ 1, 2 ... 8998, 8999, 9000, 9001, 9002 ... 9999 ]
// key [ 1001, 1002 ... 9998, 9999, 0, 1, 2 ... 999 ]
@ -165,7 +165,7 @@ TEST_F(FlushJobTest, Snapshots) {
new_mem->Ref();
SequenceNumber current_seqno = 0;
mock::MockFileContents inserted_keys;
auto inserted_keys = mock::MakeMockFile();
for (int i = 1; i < keys; ++i) {
std::string key(ToString(i));
int insertions = rnd.Uniform(max_inserts_per_keys);

49
db/internal_stats.cc
Unescape View File

@ -27,6 +27,7 @@ namespace rocksdb {
namespace {
const double kMB = 1048576.0;
const double kGB = kMB * 1024;
const double kMicrosInSec = 1000000.0;
void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
snprintf(
@ -50,7 +51,7 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
stats.bytes_read_non_output_levels + stats.bytes_read_output_level;
int64_t bytes_new =
stats.bytes_written - stats.bytes_read_output_level;
double elapsed = (stats.micros + 1) / 1000000.0;
double elapsed = (stats.micros + 1) / kMicrosInSec;
std::string num_input_records = NumberToHumanString(stats.num_input_records);
std::string num_dropped_records =
NumberToHumanString(stats.num_dropped_records);
@ -73,16 +74,15 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
" " /* Stall(cnt) */
"%7s " /* KeyIn */
"%6s\n", /* KeyDrop */
name.c_str(), num_files, being_compacted, total_file_size / kMB,
score, bytes_read / kGB, stats.bytes_read_non_output_levels / kGB,
name.c_str(),
num_files, being_compacted, total_file_size / kMB, score,
bytes_read / kGB, stats.bytes_read_non_output_levels / kGB,
stats.bytes_read_output_level / kGB, stats.bytes_written / kGB,
bytes_new / kGB, stats.bytes_moved / kGB,
w_amp, bytes_read / kMB / elapsed,
stats.bytes_written / kMB / elapsed, stats.micros / 1000000.0,
stats.count,
stats.count == 0 ? 0 : stats.micros / 1000000.0 / stats.count,
stalls,
num_input_records.c_str(), num_dropped_records.c_str());
bytes_new / kGB, stats.bytes_moved / kGB, w_amp,
bytes_read / kMB / elapsed, stats.bytes_written / kMB / elapsed,
stats.micros / kMicrosInSec, stats.count,
stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count,
stalls, num_input_records.c_str(), num_dropped_records.c_str());
}
}
@ -465,7 +465,7 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
void InternalStats::DumpDBStats(std::string* value) {
char buf[1000];
// DB-level stats, only available from default column family
double seconds_up = (env_->NowMicros() - started_at_ + 1) / 1000000.0;
double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec;
double interval_seconds_up = seconds_up - db_stats_snapshot_.seconds_up;
snprintf(buf, sizeof(buf),
"\n** DB Stats **\nUptime(secs): %.1f total, %.1f interval\n",
@ -524,11 +524,9 @@ void InternalStats::DumpDBStats(std::string* value) {
snprintf(buf, sizeof(buf),
"Cumulative compaction: %.2f GB write, %.2f MB/s write, "
"%.2f GB read, %.2f MB/s read, %.1f seconds\n",
compact_bytes_write / kGB,
compact_bytes_write / kMB / seconds_up,
compact_bytes_read / kGB,
compact_bytes_read / kMB / seconds_up,
compact_micros / 1000000.0);
compact_bytes_write / kGB, compact_bytes_write / kMB / seconds_up,
compact_bytes_read / kGB, compact_bytes_read / kMB / seconds_up,
compact_micros / kMicrosInSec);
value->append(buf);
// Stall
AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true);
@ -582,16 +580,15 @@ void InternalStats::DumpDBStats(std::string* value) {
uint64_t interval_compact_micros =
compact_micros - db_stats_snapshot_.compact_micros;
snprintf(buf, sizeof(buf),
"Interval compaction: %.2f GB write, %.2f MB/s write, "
"%.2f GB read, %.2f MB/s read, %.1f seconds\n",
interval_compact_bytes_write / kGB,
interval_compact_bytes_write / kMB /
std::max(interval_seconds_up, 0.001),
interval_compact_bytes_read / kGB,
interval_compact_bytes_read / kMB /
std::max(interval_seconds_up, 0.001),
interval_compact_micros / 1000000.0);
snprintf(
buf, sizeof(buf),
"Interval compaction: %.2f GB write, %.2f MB/s write, "
"%.2f GB read, %.2f MB/s read, %.1f seconds\n",
interval_compact_bytes_write / kGB,
interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001),
interval_compact_bytes_read / kGB,
interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001),
interval_compact_micros / kMicrosInSec);
value->append(buf);
// Stall

6
db/log_writer.h
Unescape View File

@ -6,10 +6,12 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include <memory>
#include <stdint.h>
#include <memory>
#include "db/log_format.h"
#include "rocksdb/slice.h"
#include "rocksdb/status.h"

10
db/merge_helper.cc
Unescape View File

@ -2,7 +2,6 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#include "db/merge_helper.h"
@ -92,15 +91,12 @@ Status MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
assert(!"corrupted internal key is not expected");
}
break;
}
if (user_comparator_->Compare(ikey.user_key, orig_ikey.user_key) != 0) {
} else if (user_comparator_->Compare(ikey.user_key, orig_ikey.user_key) !=
0) {
// hit a different user key, stop right here
hit_the_next_user_key = true;
break;
}
if (stop_before && ikey.sequence <= stop_before) {
} else if (stop_before && ikey.sequence <= stop_before) {
// hit an entry that's visible by the previous snapshot, can't touch that
break;
}

2
db/table_properties_collector_test.cc
Unescape View File

@ -11,8 +11,8 @@
#include "db/db_impl.h"
#include "db/dbformat.h"
#include "db/table_properties_collector.h"
#include "rocksdb/table.h"
#include "rocksdb/immutable_options.h"
#include "rocksdb/table.h"
#include "table/block_based_table_factory.h"
#include "table/meta_blocks.h"
#include "table/plain_table_factory.h"

8
db/write_batch.cc
Unescape View File

@ -27,16 +27,16 @@
#include <stack>
#include <stdexcept>
#include "rocksdb/merge_operator.h"
#include "db/dbformat.h"
#include "db/db_impl.h"
#include "db/column_family.h"
#include "db/db_impl.h"
#include "db/dbformat.h"
#include "db/memtable.h"
#include "db/snapshot_impl.h"
#include "db/write_batch_internal.h"
#include "rocksdb/merge_operator.h"
#include "util/coding.h"
#include "util/statistics.h"
#include "util/perf_context_imp.h"
#include "util/statistics.h"
namespace rocksdb {

2
include/rocksdb/utilities/write_batch_with_index.h
Unescape View File

@ -8,9 +8,9 @@
//
// A WriteBatchWithIndex with a binary searchable index built for all the keys
// inserted.
#pragma once
#ifndef ROCKSDB_LITE
#pragma once
#include <string>

2
table/meta_blocks.cc
Unescape View File

@ -8,9 +8,9 @@
#include <string>
#include "db/table_properties_collector.h"
#include "table/block.h"
#include "rocksdb/table.h"
#include "rocksdb/table_properties.h"
#include "table/block.h"
#include "table/format.h"
#include "table/table_properties_internal.h"
#include "util/coding.h"

23
table/meta_blocks.h
Unescape View File

@ -6,8 +6,8 @@
#include <map>
#include <memory>
#include <vector>
#include <string>
#include <vector>
#include "db/builder.h"
#include "db/table_properties_collector.h"
@ -16,6 +16,7 @@
#include "rocksdb/slice.h"
#include "table/block_builder.h"
#include "table/format.h"
#include "util/stl_wrappers.h"
namespace rocksdb {
@ -27,22 +28,6 @@ class Logger;
class RandomAccessFile;
struct TableProperties;
// An STL style comparator that does the bytewise comparator comparasion
// internally.
struct BytewiseLessThan {
bool operator()(const std::string& key1, const std::string& key2) const {
// smaller entries will be placed in front.
return comparator->Compare(key1, key2) <= 0;
}
const Comparator* comparator = BytewiseComparator();
};
// When writing to a block that requires entries to be sorted by
// `BytewiseComparator`, we can buffer the content to `BytewiseSortedMap`
// before writng to store.
typedef std::map<std::string, std::string, BytewiseLessThan> BytewiseSortedMap;
class MetaIndexBuilder {
public:
MetaIndexBuilder(const MetaIndexBuilder&) = delete;
@ -57,7 +42,7 @@ class MetaIndexBuilder {
private:
// store the sorted key/handle of the metablocks.
BytewiseSortedMap meta_block_handles_;
stl_wrappers::KVMap meta_block_handles_;
std::unique_ptr<BlockBuilder> meta_index_block_;
};
@ -78,7 +63,7 @@ class PropertyBlockBuilder {
private:
std::unique_ptr<BlockBuilder> properties_block_;
BytewiseSortedMap props_;
stl_wrappers::KVMap props_;
};
// Were we encounter any error occurs during user-defined statistics collection,

35
table/mock_table.cc
Unescape View File

@ -5,17 +5,29 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#include "table/mock_table.h"
#include "db/dbformat.h"
#include "port/port.h"
#include "rocksdb/table_properties.h"
#include "table/get_context.h"
#include "table/mock_table.h"
#include "util/coding.h"
#include "util/file_reader_writer.h"
namespace rocksdb {
namespace mock {
namespace {
const InternalKeyComparator icmp_(BytewiseComparator());
} // namespace
stl_wrappers::KVMap MakeMockFile(
std::initializer_list<std::pair<const std::string, std::string>> l) {
return stl_wrappers::KVMap(l, stl_wrappers::LessOfComparator(&icmp_));
}
Iterator* MockTableReader::NewIterator(const ReadOptions&, Arena* arena) {
return new MockTableIterator(table_);
}
@ -71,7 +83,7 @@ TableBuilder* MockTableFactory::NewTableBuilder(
}
Status MockTableFactory::CreateMockTable(Env* env, const std::string& fname,
MockFileContents file_contents) {
stl_wrappers::KVMap file_contents) {
std::unique_ptr<WritableFile> file;
auto s = env->NewWritableFile(fname, &file, EnvOptions());
if (!s.ok()) {
@ -99,16 +111,29 @@ uint32_t MockTableFactory::GetIDFromFile(RandomAccessFileReader* file) const {
return DecodeFixed32(buf);
}
void MockTableFactory::AssertSingleFile(const MockFileContents& file_contents) {
void MockTableFactory::AssertSingleFile(
const stl_wrappers::KVMap& file_contents) {
ASSERT_EQ(file_system_.files.size(), 1U);
ASSERT_TRUE(file_contents == file_system_.files.begin()->second);
}
void MockTableFactory::AssertLatestFile(const MockFileContents& file_contents) {
void MockTableFactory::AssertLatestFile(
const stl_wrappers::KVMap& file_contents) {
ASSERT_GE(file_system_.files.size(), 1U);
auto latest = file_system_.files.end();
--latest;
ASSERT_TRUE(file_contents == latest->second);
if (file_contents != latest->second) {
std::cout << "Wrong content! Content of latest file:" << std::endl;
for (const auto& kv : latest->second) {
ParsedInternalKey ikey;
std::string key, value;
std::tie(key, value) = kv;
ParseInternalKey(Slice(key), &ikey);
std::cout << ikey.DebugString(false) << " -> " << value << std::endl;
}
ASSERT_TRUE(false);
}
}
} // namespace mock

58
table/mock_table.h
Unescape View File

@ -5,51 +5,39 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#pragma once
#include <algorithm>
#include <set>
#include <memory>
#include <atomic>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include "port/port.h"
#include "rocksdb/comparator.h"
#include "rocksdb/table.h"
#include "table/table_reader.h"
#include "table/table_builder.h"
#include "port/port.h"
#include "table/table_reader.h"
#include "util/mutexlock.h"
#include "util/stl_wrappers.h"
#include "util/testharness.h"
#include "util/testutil.h"
namespace rocksdb {
namespace mock {
struct MockFileContentsCmp {
MockFileContentsCmp() : icmp_(BytewiseComparator()) {}
bool operator() (const std::string& x, const std::string& y) const {
InternalKey ikey_x;
InternalKey ikey_y;
ikey_x.DecodeFrom(x);
ikey_y.DecodeFrom(y);
return icmp_.Compare(ikey_x, ikey_y) < 0;
}
InternalKeyComparator icmp_;
};
// NOTE: this currently only supports the bytewise comparator
typedef std::map<std::string, std::string, MockFileContentsCmp>
MockFileContents;
stl_wrappers::KVMap MakeMockFile(
std::initializer_list<std::pair<const std::string, std::string>> l = {});
struct MockTableFileSystem {
port::Mutex mutex;
std::map<uint32_t, MockFileContents> files;
std::map<uint32_t, stl_wrappers::KVMap> files;
};
class MockTableReader : public TableReader {
public:
explicit MockTableReader(const MockFileContents& table) : table_(table) {}
explicit MockTableReader(const stl_wrappers::KVMap& table) : table_(table) {}
Iterator* NewIterator(const ReadOptions&, Arena* arena) override;
@ -67,12 +55,12 @@ class MockTableReader : public TableReader {
~MockTableReader() {}
private:
const MockFileContents& table_;
const stl_wrappers::KVMap& table_;
};
class MockTableIterator : public Iterator {
public:
explicit MockTableIterator(const MockFileContents& table) : table_(table) {
explicit MockTableIterator(const stl_wrappers::KVMap& table) : table_(table) {
itr_ = table_.end();
}
@ -107,14 +95,16 @@ class MockTableIterator : public Iterator {
Status status() const override { return Status::OK(); }
private:
const MockFileContents& table_;
MockFileContents::const_iterator itr_;
const stl_wrappers::KVMap& table_;
stl_wrappers::KVMap::const_iterator itr_;
};
class MockTableBuilder : public TableBuilder {
public:
MockTableBuilder(uint32_t id, MockTableFileSystem* file_system)
: id_(id), file_system_(file_system) {}
: id_(id), file_system_(file_system) {
table_ = MakeMockFile({});
}
// REQUIRES: Either Finish() or Abandon() has been called.
~MockTableBuilder() {}
@ -148,7 +138,7 @@ class MockTableBuilder : public TableBuilder {
private:
uint32_t id_;
MockTableFileSystem* file_system_;
MockFileContents table_;
stl_wrappers::KVMap table_;
};
class MockTableFactory : public TableFactory {
@ -166,10 +156,10 @@ class MockTableFactory : public TableFactory {
WritableFileWriter* file) const override;
// This function will directly create mock table instead of going through
// MockTableBuilder. MockFileContents has to have a format of <internal_key,
// value>. Those key-value pairs will then be inserted into the mock table
// MockTableBuilder. file_contents has to have a format of <internal_key,
// value>. Those key-value pairs will then be inserted into the mock table.
Status CreateMockTable(Env* env, const std::string& fname,
MockFileContents file_contents);
stl_wrappers::KVMap file_contents);
virtual Status SanitizeOptions(
const DBOptions& db_opts,
@ -183,8 +173,8 @@ class MockTableFactory : public TableFactory {
// This function will assert that only a single file exists and that the
// contents are equal to file_contents
void AssertSingleFile(const MockFileContents& file_contents);
void AssertLatestFile(const MockFileContents& file_contents);
void AssertSingleFile(const stl_wrappers::KVMap& file_contents);
void AssertLatestFile(const stl_wrappers::KVMap& file_contents);
private:
uint32_t GetAndWriteNextID(WritableFile* file) const;

120
table/table_test.cc
Unescape View File

@ -13,15 +13,14 @@
#include <algorithm>
#include <iostream>
#include <map>
#include <string>
#include <memory>
#include <string>
#include <vector>
#include "db/dbformat.h"
#include "db/memtable.h"
#include "db/write_batch_internal.h"
#include "db/writebuffer.h"
#include "rocksdb/cache.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
@ -29,27 +28,23 @@
#include "rocksdb/memtablerep.h"
#include "rocksdb/slice_transform.h"
#include "rocksdb/statistics.h"
#include "table/block.h"
#include "table/block_based_table_builder.h"
#include "table/block_based_table_factory.h"
#include "table/block_based_table_reader.h"
#include "table/block_builder.h"
#include "table/format.h"
#include "table/get_context.h"
#include "table/meta_blocks.h"
#include "table/plain_table_factory.h"
#include "table/get_context.h"
#include "util/compression.h"
#include "util/random.h"
#include "util/scoped_arena_iterator.h"
#include "util/statistics.h"
#include "util/stl_wrappers.h"
#include "util/string_util.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "util/scoped_arena_iterator.h"
using std::vector;
using std::string;
namespace rocksdb {
@ -106,26 +101,14 @@ void Increment(const Comparator* cmp, std::string* key) {
}
}
// An STL comparator that uses a Comparator
struct STLLessThan {
const Comparator* cmp;
STLLessThan() : cmp(BytewiseComparator()) { }
explicit STLLessThan(const Comparator* c) : cmp(c) { }
bool operator()(const std::string& a, const std::string& b) const {
return cmp->Compare(Slice(a), Slice(b)) < 0;
}
};
} // namespace
typedef std::map<std::string, std::string, STLLessThan> KVMap;
// Helper class for tests to unify the interface between
// BlockBuilder/TableBuilder and Block/Table.
class Constructor {
public:
explicit Constructor(const Comparator* cmp) : data_(STLLessThan(cmp)) {}
explicit Constructor(const Comparator* cmp)
: data_(stl_wrappers::LessOfComparator(cmp)) {}
virtual ~Constructor() { }
void Add(const std::string& key, const Slice& value) {
@ -135,18 +118,15 @@ class Constructor {
// Finish constructing the data structure with all the keys that have
// been added so far. Returns the keys in sorted order in "*keys"
// and stores the key/value pairs in "*kvmap"
void Finish(const Options& options,
const ImmutableCFOptions& ioptions,
void Finish(const Options& options, const ImmutableCFOptions& ioptions,
const BlockBasedTableOptions& table_options,
const InternalKeyComparator& internal_comparator,
std::vector<std::string>* keys, KVMap* kvmap) {
std::vector<std::string>* keys, stl_wrappers::KVMap* kvmap) {
last_internal_key_ = &internal_comparator;
*kvmap = data_;
keys->clear();
for (KVMap::const_iterator it = data_.begin();
it != data_.end();
++it) {
keys->push_back(it->first);
for (const auto& kv : data_) {
keys->push_back(kv.first);
}
data_.clear();
Status s = FinishImpl(options, ioptions, table_options,
@ -159,11 +139,11 @@ class Constructor {
const ImmutableCFOptions& ioptions,
const BlockBasedTableOptions& table_options,
const InternalKeyComparator& internal_comparator,
const KVMap& data) = 0;
const stl_wrappers::KVMap& data) = 0;
virtual Iterator* NewIterator() const = 0;
virtual const KVMap& data() { return data_; }
virtual const stl_wrappers::KVMap& data() { return data_; }
virtual bool IsArenaMode() const { return false; }
@ -175,7 +155,7 @@ class Constructor {
const InternalKeyComparator* last_internal_key_;
private:
KVMap data_;
stl_wrappers::KVMap data_;
};
class BlockConstructor: public Constructor {
@ -191,7 +171,7 @@ class BlockConstructor: public Constructor {
const ImmutableCFOptions& ioptions,
const BlockBasedTableOptions& table_options,
const InternalKeyComparator& internal_comparator,
const KVMap& kv_map) override {
const stl_wrappers::KVMap& kv_map) override {
delete block_;
block_ = nullptr;
BlockBuilder builder(table_options.block_restart_interval);
@ -280,7 +260,7 @@ class TableConstructor: public Constructor {
const ImmutableCFOptions& ioptions,
const BlockBasedTableOptions& table_options,
const InternalKeyComparator& internal_comparator,
const KVMap& kv_map) override {
const stl_wrappers::KVMap& kv_map) override {
Reset();
file_writer_.reset(test::GetWritableFileWriter(new test::StringSink()));
unique_ptr<TableBuilder> builder;
@ -393,7 +373,7 @@ class MemTableConstructor: public Constructor {
virtual Status FinishImpl(const Options&, const ImmutableCFOptions& ioptions,
const BlockBasedTableOptions& table_options,
const InternalKeyComparator& internal_comparator,
const KVMap& kv_map) override {
const stl_wrappers::KVMap& kv_map) override {
delete memtable_->Unref();
ImmutableCFOptions mem_ioptions(ioptions);
memtable_ = new MemTable(internal_comparator_, mem_ioptions,
@ -440,7 +420,7 @@ class DBConstructor: public Constructor {
const ImmutableCFOptions& ioptions,
const BlockBasedTableOptions& table_options,
const InternalKeyComparator& internal_comparator,
const KVMap& kv_map) override {
const stl_wrappers::KVMap& kv_map) override {
delete db_;
db_ = nullptr;
NewDB();
@ -705,7 +685,7 @@ class HarnessTest : public testing::Test {
void Test(Random* rnd) {
std::vector<std::string> keys;
KVMap data;
stl_wrappers::KVMap data;
constructor_->Finish(options_, ioptions_, table_options_,
*internal_comparator_, &keys, &data);
@ -717,13 +697,12 @@ class HarnessTest : public testing::Test {
}
void TestForwardScan(const std::vector<std::string>& keys,
const KVMap& data) {
const stl_wrappers::KVMap& data) {
Iterator* iter = constructor_->NewIterator();
ASSERT_TRUE(!iter->Valid());
iter->SeekToFirst();
for (KVMap::const_iterator model_iter = data.begin();
model_iter != data.end();
++model_iter) {
for (stl_wrappers::KVMap::const_iterator model_iter = data.begin();
model_iter != data.end(); ++model_iter) {
ASSERT_EQ(ToString(data, model_iter), ToString(iter));
iter->Next();
}
@ -736,13 +715,12 @@ class HarnessTest : public testing::Test {
}
void TestBackwardScan(const std::vector<std::string>& keys,
const KVMap& data) {
const stl_wrappers::KVMap& data) {
Iterator* iter = constructor_->NewIterator();
ASSERT_TRUE(!iter->Valid());
iter->SeekToLast();
for (KVMap::const_reverse_iterator model_iter = data.rbegin();
model_iter != data.rend();
++model_iter) {
for (stl_wrappers::KVMap::const_reverse_iterator model_iter = data.rbegin();
model_iter != data.rend(); ++model_iter) {
ASSERT_EQ(ToString(data, model_iter), ToString(iter));
iter->Prev();
}
@ -754,13 +732,12 @@ class HarnessTest : public testing::Test {
}
}
void TestRandomAccess(Random* rnd,
const std::vector<std::string>& keys,
const KVMap& data) {
void TestRandomAccess(Random* rnd, const std::vector<std::string>& keys,
const stl_wrappers::KVMap& data) {
static const bool kVerbose = false;
Iterator* iter = constructor_->NewIterator();
ASSERT_TRUE(!iter->Valid());
KVMap::const_iterator model_iter = data.begin();
stl_wrappers::KVMap::const_iterator model_iter = data.begin();
if (kVerbose) fprintf(stderr, "---\n");
for (int i = 0; i < 200; i++) {
const int toss = rnd->Uniform(support_prev_ ? 5 : 3);
@ -828,7 +805,8 @@ class HarnessTest : public testing::Test {
}
}
std::string ToString(const KVMap& data, const KVMap::const_iterator& it) {
std::string ToString(const stl_wrappers::KVMap& data,
const stl_wrappers::KVMap::const_iterator& it) {
if (it == data.end()) {
return "END";
} else {
@ -836,8 +814,8 @@ class HarnessTest : public testing::Test {
}
}
std::string ToString(const KVMap& data,
const KVMap::const_reverse_iterator& it) {
std::string ToString(const stl_wrappers::KVMap& data,
const stl_wrappers::KVMap::const_reverse_iterator& it) {
if (it == data.rend()) {
return "END";
} else {
@ -982,7 +960,7 @@ TEST_F(BlockBasedTableTest, BasicBlockBasedTableProperties) {
c.Add("j9", "val9");
std::vector<std::string> keys;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
Options options;
options.compression = kNoCompression;
BlockBasedTableOptions table_options;
@ -1017,7 +995,7 @@ TEST_F(BlockBasedTableTest, FilterPolicyNameProperties) {
TableConstructor c(BytewiseComparator(), true);
c.Add("a1", "val1");
std::vector<std::string> keys;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
BlockBasedTableOptions table_options;
table_options.filter_policy.reset(NewBloomFilterPolicy(10));
Options options;
@ -1034,8 +1012,8 @@ TEST_F(BlockBasedTableTest, FilterPolicyNameProperties) {
// BlockBasedTableTest::PrefetchTest
//
void AssertKeysInCache(BlockBasedTable* table_reader,
const vector<string>& keys_in_cache,
const vector<string>& keys_not_in_cache) {
const std::vector<std::string>& keys_in_cache,
const std::vector<std::string>& keys_not_in_cache) {
for (auto key : keys_in_cache) {
ASSERT_TRUE(table_reader->TEST_KeyInCache(ReadOptions(), key));
}
@ -1047,10 +1025,10 @@ void AssertKeysInCache(BlockBasedTable* table_reader,
void PrefetchRange(TableConstructor* c, Options* opt,
BlockBasedTableOptions* table_options,
const vector<std::string>& keys,
const char* key_begin, const char* key_end,
const vector<string>& keys_in_cache,
const vector<string>& keys_not_in_cache,
const std::vector<std::string>& keys, const char* key_begin,
const char* key_end,
const std::vector<std::string>& keys_in_cache,
const std::vector<std::string>& keys_not_in_cache,
const Status expected_status = Status::OK()) {
// reset the cache and reopen the table
table_options->block_cache = NewLRUCache(16 * 1024 * 1024);
@ -1093,7 +1071,7 @@ TEST_F(BlockBasedTableTest, PrefetchTest) {
c.Add("k06", "hello3");
c.Add("k07", std::string(100000, 'x'));
std::vector<std::string> keys;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
const ImmutableCFOptions ioptions(opt);
c.Finish(opt, ioptions, table_options, *ikc, &keys, &kvmap);
@ -1197,7 +1175,7 @@ TEST_F(BlockBasedTableTest, TotalOrderSeekOnHashIndex) {
c.Add("abbb1", std::string('a', 56));
c.Add("cccc2", std::string('a', 56));
std::vector<std::string> keys;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
const ImmutableCFOptions ioptions(options);
c.Finish(options, ioptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
@ -1271,7 +1249,7 @@ TEST_F(TableTest, HashIndexTest) {
AddInternalKey(&c, "0095");
std::vector<std::string> keys;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
Options options;
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
BlockBasedTableOptions table_options;
@ -1385,7 +1363,7 @@ TEST_F(BlockBasedTableTest, IndexSizeStat) {
}
std::vector<std::string> ks;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
Options options;
options.compression = kNoCompression;
BlockBasedTableOptions table_options;
@ -1418,7 +1396,7 @@ TEST_F(BlockBasedTableTest, NumBlockStat) {
}
std::vector<std::string> ks;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
const ImmutableCFOptions ioptions(options);
c.Finish(options, ioptions, table_options,
GetPlainInternalComparator(options.comparator), &ks, &kvmap);
@ -1491,7 +1469,7 @@ TEST_F(BlockBasedTableTest, BlockCacheDisabledTest) {
table_options.filter_policy.reset(NewBloomFilterPolicy(10));
options.table_factory.reset(new BlockBasedTableFactory(table_options));
std::vector<std::string> keys;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
TableConstructor c(BytewiseComparator(), true);
c.Add("key", "value");
@ -1537,7 +1515,7 @@ TEST_F(BlockBasedTableTest, FilterBlockInBlockCache) {
table_options.cache_index_and_filter_blocks = true;
options.table_factory.reset(new BlockBasedTableFactory(table_options));
std::vector<std::string> keys;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
TableConstructor c(BytewiseComparator());
c.Add("key", "value");
@ -1681,7 +1659,7 @@ TEST_F(BlockBasedTableTest, BlockCacheLeak) {
c.Add("k06", "hello3");
c.Add("k07", std::string(100000, 'x'));
std::vector<std::string> keys;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
const ImmutableCFOptions ioptions(opt);
c.Finish(opt, ioptions, table_options, *ikc, &keys, &kvmap);
@ -1775,7 +1753,7 @@ TEST_F(GeneralTableTest, ApproximateOffsetOfPlain) {
c.Add("k06", "hello3");
c.Add("k07", std::string(100000, 'x'));
std::vector<std::string> keys;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
Options options;
test::PlainInternalKeyComparator internal_comparator(options.comparator);
options.compression = kNoCompression;
@ -1807,7 +1785,7 @@ static void DoCompressionTest(CompressionType comp) {
c.Add("k03", "hello3");
c.Add("k04", test::CompressibleString(&rnd, 0.25, 10000, &tmp));
std::vector<std::string> keys;
KVMap kvmap;
stl_wrappers::KVMap kvmap;
Options options;
test::PlainInternalKeyComparator ikc(options.comparator);
options.compression = comp;

9
util/hash_cuckoo_rep.cc
Unescape View File

@ -5,20 +5,21 @@
//
#ifndef ROCKSDB_LITE
#include "util/hash_cuckoo_rep.h"
#include <algorithm>
#include <atomic>
#include <limits>
#include <memory>
#include <queue>
#include <string>
#include <memory>
#include <vector>
#include "rocksdb/memtablerep.h"
#include "util/murmurhash.h"
#include "db/memtable.h"
#include "db/skiplist.h"
#include "rocksdb/memtablerep.h"
#include "util/murmurhash.h"
#include "util/stl_wrappers.h"
namespace rocksdb {
@ -40,7 +41,7 @@ struct CuckooStep {
CuckooStep() : bucket_id_(-1), prev_step_id_(kNullStep), depth_(1) {}
// MSVC does not support = default yet
CuckooStep(CuckooStep&& o) { *this = std::move(o); }
CuckooStep(CuckooStep&& o) ROCKSDB_NOEXCEPT { *this = std::move(o); }
CuckooStep& operator=(CuckooStep&& rhs) {
bucket_id_ = std::move(rhs.bucket_id_);

48
util/stl_wrappers.h
Unescape View File

@ -2,31 +2,45 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#pragma once
#include "util/murmurhash.h"
#include "util/coding.h"
#include <map>
#include <string>
#include "rocksdb/comparator.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/slice.h"
#include "util/coding.h"
#include "util/murmurhash.h"
namespace rocksdb {
namespace stl_wrappers {
class Base {
protected:
const MemTableRep::KeyComparator& compare_;
explicit Base(const MemTableRep::KeyComparator& compare)
: compare_(compare) { }
};
struct Compare : private Base {
explicit Compare(const MemTableRep::KeyComparator& compare)
: Base(compare) { }
inline bool operator()(const char* a, const char* b) const {
return compare_(a, b) < 0;
}
};
class Base {
protected:
const MemTableRep::KeyComparator& compare_;
explicit Base(const MemTableRep::KeyComparator& compare)
: compare_(compare) {}
};
struct Compare : private Base {
explicit Compare(const MemTableRep::KeyComparator& compare) : Base(compare) {}
inline bool operator()(const char* a, const char* b) const {
return compare_(a, b) < 0;
}
};
struct LessOfComparator {
explicit LessOfComparator(const Comparator* c = BytewiseComparator())
: cmp(c) {}
bool operator()(const std::string& a, const std::string& b) const {
return cmp->Compare(Slice(a), Slice(b)) < 0;
}
const Comparator* cmp;
};
typedef std::map<std::string, std::string, LessOfComparator> KVMap;
}
}

6
utilities/write_batch_with_index/write_batch_with_index.cc
Unescape View File

@ -7,15 +7,15 @@
#include "rocksdb/utilities/write_batch_with_index.h"
#include <memory>
#include <limits>
#include <memory>
#include "rocksdb/comparator.h"
#include "rocksdb/iterator.h"
#include "db/column_family.h"
#include "db/merge_context.h"
#include "db/merge_helper.h"
#include "db/skiplist.h"
#include "rocksdb/comparator.h"
#include "rocksdb/iterator.h"
#include "util/arena.h"
#include "utilities/write_batch_with_index/write_batch_with_index_internal.h"

3
utilities/write_batch_with_index/write_batch_with_index_internal.h
Unescape View File

@ -2,11 +2,10 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#pragma once
#ifndef ROCKSDB_LITE
#pragma once
#include <limits>
#include <string>
#include <unordered_map>

Write Preview
Loading…
Cancel
Save