NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Apply formatter on recent 45 commits. (#5827)

Browse Source 
Summary:
Some recent commits might not have passed through the formatter. I formatted recent 45 commits. The script hangs for more commits so I stopped there.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5827

Test Plan: Run all existing tests.

Differential Revision: D17483727

fbshipit-source-id: af23113ee63015d8a43d89a3bc2c1056189afe8f
main
sdong 5 years ago committed by Facebook Github Bot
parent 6ec6a4a9a4
commit c06b54d0c6
26 changed files with 270 additions and 279 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      db/arena_wrapped_db_iter.cc
  2. 2
      db/compaction/compaction_iterator.cc
  3. 50
      db/compaction/compaction_iterator.h
  4. 14
      db/compaction/compaction_job.cc
  5. 32
      db/compaction/compaction_job.h
  6. 17
      db/db_impl/db_impl_compaction_flush.cc
  7. 2
      db/db_impl/db_impl_readonly.cc
  8. 9
      db/db_iter.h
  9. 16
      db/db_test2.cc
  10. 42
      java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java
  11. 111
      java/rocksjni/sst_file_reader_iterator.cc
  12. 44
      java/rocksjni/sst_file_readerjni.cc
  13. 25
      java/src/main/java/org/rocksdb/SstFileReader.java
  14. 4
      java/src/main/java/org/rocksdb/SstFileReaderIterator.java
  15. 18
      java/src/test/java/org/rocksdb/SstFileReaderTest.java
  16. 3
      options/options_settable_test.cc
  17. 6
      tools/block_cache_analyzer/block_cache_trace_analyzer.cc
  18. 5
      tools/db_stress.cc
  19. 5
      tools/sst_dump_tool.cc
  20. 30
      util/bloom_impl.h
  21. 4
      util/concurrent_task_limiter_impl.h
  22. 26
      util/crc32c_arm64.h
  23. 2
      util/dynamic_bloom.cc
  24. 66
      util/dynamic_bloom_test.cc
  25. 12
      util/mutexlock.h
  26. 2
      utilities/transactions/write_unprepared_txn_db.cc

2
db/arena_wrapped_db_iter.cc
Unescape View File

@ -19,7 +19,7 @@
namespace rocksdb { namespace rocksdb {
Status ArenaWrappedDBIter::GetProperty(std::string prop_name, Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
std::string* prop) { std::string* prop) {
if (prop_name == "rocksdb.iterator.super-version-number") { if (prop_name == "rocksdb.iterator.super-version-number") {
// First try to pass the value returned from inner iterator. // First try to pass the value returned from inner iterator.
if (!db_iter_->GetProperty(prop_name, prop).ok()) { if (!db_iter_->GetProperty(prop_name, prop).ok()) {

2
db/compaction/compaction_iterator.cc
Unescape View File

@ -230,7 +230,7 @@ void CompactionIterator::NextFromInput() {
valid_ = false; valid_ = false;
while (!valid_ && input_->Valid() && !IsPausingManualCompaction() && while (!valid_ && input_->Valid() && !IsPausingManualCompaction() &&
!IsShuttingDown()) { !IsShuttingDown()) {
key_ = input_->key(); key_ = input_->key();
value_ = input_->value(); value_ = input_->value();
iter_stats_.num_input_records++; iter_stats_.num_input_records++;

50
db/compaction/compaction_iterator.h
Unescape View File

@ -59,32 +59,32 @@ class CompactionIterator {
const Compaction* compaction_; const Compaction* compaction_;
}; };
CompactionIterator(InternalIterator* input, const Comparator* cmp, CompactionIterator(
MergeHelper* merge_helper, SequenceNumber last_sequence, InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
std::vector<SequenceNumber>* snapshots, SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
SequenceNumber earliest_write_conflict_snapshot, SequenceNumber earliest_write_conflict_snapshot,
const SnapshotChecker* snapshot_checker, Env* env, const SnapshotChecker* snapshot_checker, Env* env,
bool report_detailed_time, bool expect_valid_internal_key, bool report_detailed_time, bool expect_valid_internal_key,
CompactionRangeDelAggregator* range_del_agg, CompactionRangeDelAggregator* range_del_agg,
const Compaction* compaction = nullptr, const Compaction* compaction = nullptr,
const CompactionFilter* compaction_filter = nullptr, const CompactionFilter* compaction_filter = nullptr,
const std::atomic<bool>* shutting_down = nullptr, const std::atomic<bool>* shutting_down = nullptr,
const SequenceNumber preserve_deletes_seqnum = 0, const SequenceNumber preserve_deletes_seqnum = 0,
const std::atomic<bool>* manual_compaction_paused = nullptr); const std::atomic<bool>* manual_compaction_paused = nullptr);
// Constructor with custom CompactionProxy, used for tests. // Constructor with custom CompactionProxy, used for tests.
CompactionIterator(InternalIterator* input, const Comparator* cmp, CompactionIterator(
MergeHelper* merge_helper, SequenceNumber last_sequence, InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
std::vector<SequenceNumber>* snapshots, SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
SequenceNumber earliest_write_conflict_snapshot, SequenceNumber earliest_write_conflict_snapshot,
const SnapshotChecker* snapshot_checker, Env* env, const SnapshotChecker* snapshot_checker, Env* env,
bool report_detailed_time, bool expect_valid_internal_key, bool report_detailed_time, bool expect_valid_internal_key,
CompactionRangeDelAggregator* range_del_agg, CompactionRangeDelAggregator* range_del_agg,
std::unique_ptr<CompactionProxy> compaction, std::unique_ptr<CompactionProxy> compaction,
const CompactionFilter* compaction_filter = nullptr, const CompactionFilter* compaction_filter = nullptr,
const std::atomic<bool>* shutting_down = nullptr, const std::atomic<bool>* shutting_down = nullptr,
const SequenceNumber preserve_deletes_seqnum = 0, const SequenceNumber preserve_deletes_seqnum = 0,
const std::atomic<bool>* manual_compaction_paused = nullptr); const std::atomic<bool>* manual_compaction_paused = nullptr);
~CompactionIterator(); ~CompactionIterator();
@ -231,7 +231,7 @@ class CompactionIterator {
bool IsPausingManualCompaction() { bool IsPausingManualCompaction() {
// This is a best-effort facility, so memory_order_relaxed is sufficient. // This is a best-effort facility, so memory_order_relaxed is sufficient.
return manual_compaction_paused_ && return manual_compaction_paused_ &&
manual_compaction_paused_->load(std::memory_order_relaxed); manual_compaction_paused_->load(std::memory_order_relaxed);
} }
}; };
} // namespace rocksdb } // namespace rocksdb

14
db/compaction/compaction_job.cc
Unescape View File

@ -870,9 +870,10 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
db_options_.statistics.get()); db_options_.statistics.get());
TEST_SYNC_POINT("CompactionJob::Run():Inprogress"); TEST_SYNC_POINT("CompactionJob::Run():Inprogress");
TEST_SYNC_POINT_CALLBACK("CompactionJob::Run():PausingManualCompaction:1", TEST_SYNC_POINT_CALLBACK(
reinterpret_cast<void *>( "CompactionJob::Run():PausingManualCompaction:1",
const_cast<std::atomic<bool> *>(manual_compaction_paused_))); reinterpret_cast<void*>(
const_cast<std::atomic<bool>*>(manual_compaction_paused_)));
Slice* start = sub_compact->start; Slice* start = sub_compact->start;
Slice* end = sub_compact->end; Slice* end = sub_compact->end;
@ -954,9 +955,10 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
input_status = input->status(); input_status = input->status();
output_file_ended = true; output_file_ended = true;
} }
TEST_SYNC_POINT_CALLBACK("CompactionJob::Run():PausingManualCompaction:2", TEST_SYNC_POINT_CALLBACK(
reinterpret_cast<void *>( "CompactionJob::Run():PausingManualCompaction:2",
const_cast<std::atomic<bool> *>(manual_compaction_paused_))); reinterpret_cast<void*>(
const_cast<std::atomic<bool>*>(manual_compaction_paused_)));
c_iter->Next(); c_iter->Next();
if (c_iter->status().IsManualCompactionPaused()) { if (c_iter->status().IsManualCompactionPaused()) {
break; break;

32
db/compaction/compaction_job.h
Unescape View File

@ -62,21 +62,23 @@ class VersionSet;
// if needed. // if needed.
class CompactionJob { class CompactionJob {
public: public:
CompactionJob( CompactionJob(int job_id, Compaction* compaction,
int job_id, Compaction* compaction, const ImmutableDBOptions& db_options, const ImmutableDBOptions& db_options,
const EnvOptions env_options, VersionSet* versions, const EnvOptions env_options, VersionSet* versions,
const std::atomic<bool>* shutting_down, const std::atomic<bool>* shutting_down,
const SequenceNumber preserve_deletes_seqnum, LogBuffer* log_buffer, const SequenceNumber preserve_deletes_seqnum,
Directory* db_directory, Directory* output_directory, Statistics* stats, LogBuffer* log_buffer, Directory* db_directory,
InstrumentedMutex* db_mutex, ErrorHandler* db_error_handler, Directory* output_directory, Statistics* stats,
std::vector<SequenceNumber> existing_snapshots, InstrumentedMutex* db_mutex, ErrorHandler* db_error_handler,
SequenceNumber earliest_write_conflict_snapshot, std::vector<SequenceNumber> existing_snapshots,
const SnapshotChecker* snapshot_checker, SequenceNumber earliest_write_conflict_snapshot,
std::shared_ptr<Cache> table_cache, EventLogger* event_logger, const SnapshotChecker* snapshot_checker,
bool paranoid_file_checks, bool measure_io_stats, std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
const std::string& dbname, CompactionJobStats* compaction_job_stats, bool paranoid_file_checks, bool measure_io_stats,
Env::Priority thread_pri, const std::string& dbname,
const std::atomic<bool>* manual_compaction_paused = nullptr); CompactionJobStats* compaction_job_stats,
Env::Priority thread_pri,
const std::atomic<bool>* manual_compaction_paused = nullptr);
~CompactionJob(); ~CompactionJob();

17
db/db_impl/db_impl_compaction_flush.cc
Unescape View File

@ -983,8 +983,7 @@ Status DBImpl::CompactFilesImpl(
snapshot_checker, table_cache_, &event_logger_, snapshot_checker, table_cache_, &event_logger_,
c->mutable_cf_options()->paranoid_file_checks, c->mutable_cf_options()->paranoid_file_checks,
c->mutable_cf_options()->report_bg_io_stats, dbname_, c->mutable_cf_options()->report_bg_io_stats, dbname_,
&compaction_job_stats, Env::Priority::USER, &compaction_job_stats, Env::Priority::USER, &manual_compaction_paused_);
&manual_compaction_paused_);
// Creating a compaction influences the compaction score because the score // Creating a compaction influences the compaction score because the score
// takes running compactions into account (by skipping files that are already // takes running compactions into account (by skipping files that are already
@ -2313,8 +2312,7 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
env_->SleepForMicroseconds(10000); // prevent hot loop env_->SleepForMicroseconds(10000); // prevent hot loop
mutex_.Lock(); mutex_.Lock();
} else if (!s.ok() && !s.IsShutdownInProgress() && } else if (!s.ok() && !s.IsShutdownInProgress() &&
!s.IsManualCompactionPaused() && !s.IsManualCompactionPaused() && !s.IsColumnFamilyDropped()) {
!s.IsColumnFamilyDropped()) {
// Wait a little bit before retrying background compaction in // Wait a little bit before retrying background compaction in
// case this is an environmental problem and we do not want to // case this is an environmental problem and we do not want to
// chew up resources for failed compactions for the duration of // chew up resources for failed compactions for the duration of
@ -2332,11 +2330,10 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
env_->SleepForMicroseconds(1000000); env_->SleepForMicroseconds(1000000);
mutex_.Lock(); mutex_.Lock();
} else if (s.IsManualCompactionPaused()) { } else if (s.IsManualCompactionPaused()) {
ManualCompactionState *m = prepicked_compaction->manual_compaction_state; ManualCompactionState* m = prepicked_compaction->manual_compaction_state;
assert(m); assert(m);
ROCKS_LOG_BUFFER(&log_buffer, "[%s] [JOB %d] Manual compaction paused", ROCKS_LOG_BUFFER(&log_buffer, "[%s] [JOB %d] Manual compaction paused",
m->cfd->GetName().c_str(), m->cfd->GetName().c_str(), job_context.job_id);
job_context.job_id);
} }
ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem); ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
@ -2345,8 +2342,8 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
// have created (they might not be all recorded in job_context in case of a // have created (they might not be all recorded in job_context in case of a
// failure). Thus, we force full scan in FindObsoleteFiles() // failure). Thus, we force full scan in FindObsoleteFiles()
FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress() && FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress() &&
!s.IsManualCompactionPaused() && !s.IsManualCompactionPaused() &&
!s.IsColumnFamilyDropped()); !s.IsColumnFamilyDropped());
TEST_SYNC_POINT("DBImpl::BackgroundCallCompaction:FoundObsoleteFiles"); TEST_SYNC_POINT("DBImpl::BackgroundCallCompaction:FoundObsoleteFiles");
// delete unnecessary files if any, this is done outside the mutex // delete unnecessary files if any, this is done outside the mutex
@ -2430,7 +2427,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
if (shutting_down_.load(std::memory_order_acquire)) { if (shutting_down_.load(std::memory_order_acquire)) {
status = Status::ShutdownInProgress(); status = Status::ShutdownInProgress();
} else if (is_manual && } else if (is_manual &&
manual_compaction_paused_.load(std::memory_order_acquire)) { manual_compaction_paused_.load(std::memory_order_acquire)) {
status = Status::Incomplete(Status::SubCode::kManualCompactionPaused); status = Status::Incomplete(Status::SubCode::kManualCompactionPaused);
} }
} else { } else {

2
db/db_impl/db_impl_readonly.cc
Unescape View File

@ -3,8 +3,8 @@
// COPYING file in the root directory) and Apache 2.0 License // COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#include "db/arena_wrapped_db_iter.h"
#include "db/db_impl/db_impl_readonly.h" #include "db/db_impl/db_impl_readonly.h"
#include "db/arena_wrapped_db_iter.h"
#include "db/compacted_db_impl.h" #include "db/compacted_db_impl.h"
#include "db/db_impl/db_impl.h" #include "db/db_impl/db_impl.h"

9
db/db_iter.h
Unescape View File

@ -53,7 +53,7 @@ namespace rocksdb {
// combines multiple entries for the same userkey found in the DB // combines multiple entries for the same userkey found in the DB
// representation into a single entry while accounting for sequence // representation into a single entry while accounting for sequence
// numbers, deletion markers, overwrites, etc. // numbers, deletion markers, overwrites, etc.
class DBIter final: public Iterator { class DBIter final : public Iterator {
public: public:
// The following is grossly complicated. TODO: clean it up // The following is grossly complicated. TODO: clean it up
// Which direction is the iterator currently moving? // Which direction is the iterator currently moving?
@ -66,10 +66,7 @@ class DBIter final: public Iterator {
// this->key(). // this->key().
// (2) When moving backwards, the internal iterator is positioned // (2) When moving backwards, the internal iterator is positioned
// just before all entries whose user key == this->key(). // just before all entries whose user key == this->key().
enum Direction { enum Direction { kForward, kReverse };
kForward,
kReverse
};
// LocalStatistics contain Statistics counters that will be aggregated per // LocalStatistics contain Statistics counters that will be aggregated per
// each iterator instance and then will be sent to the global statistics when // each iterator instance and then will be sent to the global statistics when
@ -148,7 +145,7 @@ class DBIter final: public Iterator {
bool Valid() const override { return valid_; } bool Valid() const override { return valid_; }
Slice key() const override { Slice key() const override {
assert(valid_); assert(valid_);
if(start_seqnum_ > 0) { if (start_seqnum_ > 0) {
return saved_key_.GetInternalKey(); return saved_key_.GetInternalKey();
} else { } else {
return saved_key_.GetUserKey(); return saved_key_.GetUserKey();

16
db/db_test2.cc
Unescape View File

@ -2458,8 +2458,7 @@ TEST_F(DBTest2, PausingManualCompaction1) {
manual_compactions_paused = 0; manual_compactions_paused = 0;
// Now make sure CompactFiles also not run // Now make sure CompactFiles also not run
dbfull()->CompactFiles(rocksdb::CompactionOptions(), dbfull()->CompactFiles(rocksdb::CompactionOptions(), files_before_compact, 0);
files_before_compact, 0);
// Wait for manual compaction to get scheduled and finish // Wait for manual compaction to get scheduled and finish
dbfull()->TEST_WaitForCompact(true); dbfull()->TEST_WaitForCompact(true);
@ -2510,14 +2509,14 @@ TEST_F(DBTest2, PausingManualCompaction3) {
Random rnd(301); Random rnd(301);
auto generate_files = [&]() { auto generate_files = [&]() {
for (int i = 0; i < options.num_levels; i++) { for (int i = 0; i < options.num_levels; i++) {
for (int j = 0; j < options.num_levels-i+1; j++) { for (int j = 0; j < options.num_levels - i + 1; j++) {
for (int k = 0; k < 1000; k++) { for (int k = 0; k < 1000; k++) {
ASSERT_OK(Put(Key(k + j * 1000), RandomString(&rnd, 50))); ASSERT_OK(Put(Key(k + j * 1000), RandomString(&rnd, 50)));
} }
Flush(); Flush();
} }
for (int l = 1; l < options.num_levels-i; l++) { for (int l = 1; l < options.num_levels - i; l++) {
MoveFilesToLevel(l); MoveFilesToLevel(l);
} }
} }
@ -2530,9 +2529,8 @@ TEST_F(DBTest2, PausingManualCompaction3) {
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
int run_manual_compactions = 0; int run_manual_compactions = 0;
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
"CompactionJob::Run():PausingManualCompaction:1", [&](void* /*arg*/) { "CompactionJob::Run():PausingManualCompaction:1",
run_manual_compactions++; [&](void* /*arg*/) { run_manual_compactions++; });
});
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); rocksdb::SyncPoint::GetInstance()->EnableProcessing();
dbfull()->DisableManualCompaction(); dbfull()->DisableManualCompaction();
@ -2565,14 +2563,14 @@ TEST_F(DBTest2, PausingManualCompaction4) {
Random rnd(301); Random rnd(301);
auto generate_files = [&]() { auto generate_files = [&]() {
for (int i = 0; i < options.num_levels; i++) { for (int i = 0; i < options.num_levels; i++) {
for (int j = 0; j < options.num_levels-i+1; j++) { for (int j = 0; j < options.num_levels - i + 1; j++) {
for (int k = 0; k < 1000; k++) { for (int k = 0; k < 1000; k++) {
ASSERT_OK(Put(Key(k + j * 1000), RandomString(&rnd, 50))); ASSERT_OK(Put(Key(k + j * 1000), RandomString(&rnd, 50)));
} }
Flush(); Flush();
} }
for (int l = 1; l < options.num_levels-i; l++) { for (int l = 1; l < options.num_levels - i; l++) {
MoveFilesToLevel(l); MoveFilesToLevel(l);
} }
} }

42
java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java
Unescape View File

@ -646,8 +646,8 @@ public class DbBenchmark {
currentTaskId++, randSeed_, num_, num_, writeOpt, 1)); currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
break; break;
case "fillbatch": case "fillbatch":
tasks.add(new WriteSequentialTask( tasks.add(
currentTaskId++, randSeed_, num_, num_, writeOpt, 1000)); new WriteSequentialTask(currentTaskId++, randSeed_, num_, num_, writeOpt, 1000));
break; break;
case "fillrandom": case "fillrandom":
tasks.add(new WriteRandomTask( tasks.add(new WriteRandomTask(
@ -901,27 +901,23 @@ public class DbBenchmark {
} }
private enum Flag { private enum Flag {
benchmarks( benchmarks(Arrays.asList("fillseq", "readrandom", "fillrandom"),
Arrays.asList( "Comma-separated list of operations to run in the specified order\n"
"fillseq", + "\tActual benchmarks:\n"
"readrandom", + "\t\tfillseq -- write N values in sequential key order in async mode.\n"
"fillrandom"), + "\t\tfillrandom -- write N values in random key order in async mode.\n"
"Comma-separated list of operations to run in the specified order\n" + + "\t\tfillbatch -- write N/1000 batch where each batch has 1000 values\n"
"\tActual benchmarks:\n" + + "\t\t in sequential key order in sync mode.\n"
"\t\tfillseq -- write N values in sequential key order in async mode.\n" + + "\t\tfillsync -- write N/100 values in random key order in sync mode.\n"
"\t\tfillrandom -- write N values in random key order in async mode.\n" + + "\t\tfill100K -- write N/1000 100K values in random order in async mode.\n"
"\t\tfillbatch -- write N/1000 batch where each batch has 1000 values\n" + + "\t\treadseq -- read N times sequentially.\n"
"\t\t in sequential key order in sync mode.\n" + + "\t\treadrandom -- read N times in random order.\n"
"\t\tfillsync -- write N/100 values in random key order in sync mode.\n" + + "\t\treadhot -- read N times in random order from 1% section of DB.\n"
"\t\tfill100K -- write N/1000 100K values in random order in async mode.\n" + + "\t\treadwhilewriting -- measure the read performance of multiple readers\n"
"\t\treadseq -- read N times sequentially.\n" + + "\t\t with a bg single writer. The write rate of the bg\n"
"\t\treadrandom -- read N times in random order.\n" + + "\t\t is capped by --writes_per_second.\n"
"\t\treadhot -- read N times in random order from 1% section of DB.\n" + + "\tMeta Operations:\n"
"\t\treadwhilewriting -- measure the read performance of multiple readers\n" + + "\t\tdelete -- delete DB") {
"\t\t with a bg single writer. The write rate of the bg\n" +
"\t\t is capped by --writes_per_second.\n" +
"\tMeta Operations:\n" +
"\t\tdelete -- delete DB") {
@Override public Object parseValue(String value) { @Override public Object parseValue(String value) {
return new ArrayList<String>(Arrays.asList(value.split(","))); return new ArrayList<String>(Arrays.asList(value.split(",")));
} }

111
java/rocksjni/sst_file_reader_iterator.cc
Unescape View File

@ -6,13 +6,13 @@
// This file implements the "bridge" between Java and C++ and enables // This file implements the "bridge" between Java and C++ and enables
// calling c++ rocksdb::Iterator methods from Java side. // calling c++ rocksdb::Iterator methods from Java side.
#include <jni.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <jni.h>
#include "include/org_rocksdb_SstFileReaderIterator.h" #include "include/org_rocksdb_SstFileReaderIterator.h"
#include "rocksjni/portal.h"
#include "rocksdb/iterator.h" #include "rocksdb/iterator.h"
#include "rocksjni/portal.h"
/* /*
* Class: org_rocksdb_SstFileReaderIterator * Class: org_rocksdb_SstFileReaderIterator
@ -22,9 +22,9 @@
void Java_org_rocksdb_SstFileReaderIterator_disposeInternal(JNIEnv* /*env*/, void Java_org_rocksdb_SstFileReaderIterator_disposeInternal(JNIEnv* /*env*/,
jobject /*jobj*/, jobject /*jobj*/,
jlong handle) { jlong handle) {
auto* it = reinterpret_cast<rocksdb::Iterator*>(handle); auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
assert(it != nullptr); assert(it != nullptr);
delete it; delete it;
} }
/* /*
@ -35,7 +35,7 @@ void Java_org_rocksdb_SstFileReaderIterator_disposeInternal(JNIEnv* /*env*/,
jboolean Java_org_rocksdb_SstFileReaderIterator_isValid0(JNIEnv* /*env*/, jboolean Java_org_rocksdb_SstFileReaderIterator_isValid0(JNIEnv* /*env*/,
jobject /*jobj*/, jobject /*jobj*/,
jlong handle) { jlong handle) {
return reinterpret_cast<rocksdb::Iterator*>(handle)->Valid(); return reinterpret_cast<rocksdb::Iterator*>(handle)->Valid();
} }
/* /*
@ -46,7 +46,7 @@ jboolean Java_org_rocksdb_SstFileReaderIterator_isValid0(JNIEnv* /*env*/,
void Java_org_rocksdb_SstFileReaderIterator_seekToFirst0(JNIEnv* /*env*/, void Java_org_rocksdb_SstFileReaderIterator_seekToFirst0(JNIEnv* /*env*/,
jobject /*jobj*/, jobject /*jobj*/,
jlong handle) { jlong handle) {
reinterpret_cast<rocksdb::Iterator*>(handle)->SeekToFirst(); reinterpret_cast<rocksdb::Iterator*>(handle)->SeekToFirst();
} }
/* /*
@ -57,7 +57,7 @@ void Java_org_rocksdb_SstFileReaderIterator_seekToFirst0(JNIEnv* /*env*/,
void Java_org_rocksdb_SstFileReaderIterator_seekToLast0(JNIEnv* /*env*/, void Java_org_rocksdb_SstFileReaderIterator_seekToLast0(JNIEnv* /*env*/,
jobject /*jobj*/, jobject /*jobj*/,
jlong handle) { jlong handle) {
reinterpret_cast<rocksdb::Iterator*>(handle)->SeekToLast(); reinterpret_cast<rocksdb::Iterator*>(handle)->SeekToLast();
} }
/* /*
@ -65,9 +65,10 @@ void Java_org_rocksdb_SstFileReaderIterator_seekToLast0(JNIEnv* /*env*/,
* Method: next0 * Method: next0
* Signature: (J)V * Signature: (J)V
*/ */
void Java_org_rocksdb_SstFileReaderIterator_next0(JNIEnv* /*env*/, jobject /*jobj*/, void Java_org_rocksdb_SstFileReaderIterator_next0(JNIEnv* /*env*/,
jobject /*jobj*/,
jlong handle) { jlong handle) {
reinterpret_cast<rocksdb::Iterator*>(handle)->Next(); reinterpret_cast<rocksdb::Iterator*>(handle)->Next();
} }
/* /*
@ -75,9 +76,10 @@ void Java_org_rocksdb_SstFileReaderIterator_next0(JNIEnv* /*env*/, jobject /*job
* Method: prev0 * Method: prev0
* Signature: (J)V * Signature: (J)V
*/ */
void Java_org_rocksdb_SstFileReaderIterator_prev0(JNIEnv* /*env*/, jobject /*jobj*/, void Java_org_rocksdb_SstFileReaderIterator_prev0(JNIEnv* /*env*/,
jobject /*jobj*/,
jlong handle) { jlong handle) {
reinterpret_cast<rocksdb::Iterator*>(handle)->Prev(); reinterpret_cast<rocksdb::Iterator*>(handle)->Prev();
} }
/* /*
@ -86,21 +88,21 @@ void Java_org_rocksdb_SstFileReaderIterator_prev0(JNIEnv* /*env*/, jobject /*job
* Signature: (J[BI)V * Signature: (J[BI)V
*/ */
void Java_org_rocksdb_SstFileReaderIterator_seek0(JNIEnv* env, jobject /*jobj*/, void Java_org_rocksdb_SstFileReaderIterator_seek0(JNIEnv* env, jobject /*jobj*/,
jlong handle, jbyteArray jtarget, jlong handle,
jbyteArray jtarget,
jint jtarget_len) { jint jtarget_len) {
jbyte* target = env->GetByteArrayElements(jtarget, nullptr); jbyte* target = env->GetByteArrayElements(jtarget, nullptr);
if(target == nullptr) { if (target == nullptr) {
// exception thrown: OutOfMemoryError // exception thrown: OutOfMemoryError
return; return;
} }
rocksdb::Slice target_slice( rocksdb::Slice target_slice(reinterpret_cast<char*>(target), jtarget_len);
reinterpret_cast<char*>(target), jtarget_len);
auto* it = reinterpret_cast<rocksdb::Iterator*>(handle); auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
it->Seek(target_slice); it->Seek(target_slice);
env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT); env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT);
} }
/* /*
@ -108,23 +110,23 @@ void Java_org_rocksdb_SstFileReaderIterator_seek0(JNIEnv* env, jobject /*jobj*/,
* Method: seekForPrev0 * Method: seekForPrev0
* Signature: (J[BI)V * Signature: (J[BI)V
*/ */
void Java_org_rocksdb_SstFileReaderIterator_seekForPrev0(JNIEnv* env, jobject /*jobj*/, void Java_org_rocksdb_SstFileReaderIterator_seekForPrev0(JNIEnv* env,
jobject /*jobj*/,
jlong handle, jlong handle,
jbyteArray jtarget, jbyteArray jtarget,
jint jtarget_len) { jint jtarget_len) {
jbyte* target = env->GetByteArrayElements(jtarget, nullptr); jbyte* target = env->GetByteArrayElements(jtarget, nullptr);
if(target == nullptr) { if (target == nullptr) {
// exception thrown: OutOfMemoryError // exception thrown: OutOfMemoryError
return; return;
} }
rocksdb::Slice target_slice( rocksdb::Slice target_slice(reinterpret_cast<char*>(target), jtarget_len);
reinterpret_cast<char*>(target), jtarget_len);
auto* it = reinterpret_cast<rocksdb::Iterator*>(handle); auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
it->SeekForPrev(target_slice); it->SeekForPrev(target_slice);
env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT); env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT);
} }
/* /*
@ -132,16 +134,17 @@ void Java_org_rocksdb_SstFileReaderIterator_seekForPrev0(JNIEnv* env, jobject /*
* Method: status0 * Method: status0
* Signature: (J)V * Signature: (J)V
*/ */
void Java_org_rocksdb_SstFileReaderIterator_status0(JNIEnv* env, jobject /*jobj*/, void Java_org_rocksdb_SstFileReaderIterator_status0(JNIEnv* env,
jobject /*jobj*/,
jlong handle) { jlong handle) {
auto* it = reinterpret_cast<rocksdb::Iterator*>(handle); auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
rocksdb::Status s = it->status(); rocksdb::Status s = it->status();
if (s.ok()) { if (s.ok()) {
return; return;
} }
rocksdb::RocksDBExceptionJni::ThrowNew(env, s); rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
} }
/* /*
@ -149,19 +152,21 @@ void Java_org_rocksdb_SstFileReaderIterator_status0(JNIEnv* env, jobject /*jobj*
* Method: key0 * Method: key0
* Signature: (J)[B * Signature: (J)[B
*/ */
jbyteArray Java_org_rocksdb_SstFileReaderIterator_key0(JNIEnv* env, jobject /*jobj*/, jbyteArray Java_org_rocksdb_SstFileReaderIterator_key0(JNIEnv* env,
jobject /*jobj*/,
jlong handle) { jlong handle) {
auto* it = reinterpret_cast<rocksdb::Iterator*>(handle); auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
rocksdb::Slice key_slice = it->key(); rocksdb::Slice key_slice = it->key();
jbyteArray jkey = env->NewByteArray(static_cast<jsize>(key_slice.size())); jbyteArray jkey = env->NewByteArray(static_cast<jsize>(key_slice.size()));
if(jkey == nullptr) { if (jkey == nullptr) {
// exception thrown: OutOfMemoryError // exception thrown: OutOfMemoryError
return nullptr; return nullptr;
} }
env->SetByteArrayRegion(jkey, 0, static_cast<jsize>(key_slice.size()), env->SetByteArrayRegion(
const_cast<jbyte*>(reinterpret_cast<const jbyte*>(key_slice.data()))); jkey, 0, static_cast<jsize>(key_slice.size()),
return jkey; const_cast<jbyte*>(reinterpret_cast<const jbyte*>(key_slice.data())));
return jkey;
} }
/* /*

44
java/rocksjni/sst_file_readerjni.cc
Unescape View File

@ -23,8 +23,8 @@
* Signature: (J)J * Signature: (J)J
*/ */
jlong Java_org_rocksdb_SstFileReader_newSstFileReader(JNIEnv * /*env*/, jlong Java_org_rocksdb_SstFileReader_newSstFileReader(JNIEnv * /*env*/,
jclass /*jcls*/, jclass /*jcls*/,
jlong joptions) { jlong joptions) {
auto *options = reinterpret_cast<const rocksdb::Options *>(joptions); auto *options = reinterpret_cast<const rocksdb::Options *>(joptions);
rocksdb::SstFileReader *sst_file_reader = rocksdb::SstFileReader *sst_file_reader =
new rocksdb::SstFileReader(*options); new rocksdb::SstFileReader(*options);
@ -53,18 +53,18 @@ void Java_org_rocksdb_SstFileReader_open(JNIEnv *env, jobject /*jobj*/,
} }
/* /*
* Class: org_rocksdb_SstFileReader * Class: org_rocksdb_SstFileReader
* Method: newIterator * Method: newIterator
* Signature: (JJ)J * Signature: (JJ)J
*/ */
jlong Java_org_rocksdb_SstFileReader_newIterator(JNIEnv* /*env*/, jlong Java_org_rocksdb_SstFileReader_newIterator(JNIEnv * /*env*/,
jobject /*jobj*/, jobject /*jobj*/,
jlong jhandle, jlong jhandle,
jlong jread_options_handle) { jlong jread_options_handle) {
auto* sst_file_reader = reinterpret_cast<rocksdb::SstFileReader*>(jhandle); auto *sst_file_reader = reinterpret_cast<rocksdb::SstFileReader *>(jhandle);
auto* read_options = auto *read_options =
reinterpret_cast<rocksdb::ReadOptions*>(jread_options_handle); reinterpret_cast<rocksdb::ReadOptions *>(jread_options_handle);
return reinterpret_cast<jlong>(sst_file_reader->NewIterator(*read_options)); return reinterpret_cast<jlong>(sst_file_reader->NewIterator(*read_options));
} }
/* /*
@ -75,7 +75,7 @@ jlong Java_org_rocksdb_SstFileReader_newIterator(JNIEnv* /*env*/,
void Java_org_rocksdb_SstFileReader_disposeInternal(JNIEnv * /*env*/, void Java_org_rocksdb_SstFileReader_disposeInternal(JNIEnv * /*env*/,
jobject /*jobj*/, jobject /*jobj*/,
jlong jhandle) { jlong jhandle) {
delete reinterpret_cast<rocksdb::SstFileReader *>(jhandle); delete reinterpret_cast<rocksdb::SstFileReader *>(jhandle);
} }
/* /*
@ -84,9 +84,9 @@ void Java_org_rocksdb_SstFileReader_disposeInternal(JNIEnv * /*env*/,
* Signature: (J)V * Signature: (J)V
*/ */
void Java_org_rocksdb_SstFileReader_verifyChecksum(JNIEnv *env, void Java_org_rocksdb_SstFileReader_verifyChecksum(JNIEnv *env,
jobject /*jobj*/, jobject /*jobj*/,
jlong jhandle) { jlong jhandle) {
auto* sst_file_reader = reinterpret_cast<rocksdb::SstFileReader*>(jhandle); auto *sst_file_reader = reinterpret_cast<rocksdb::SstFileReader *>(jhandle);
auto s = sst_file_reader->VerifyChecksum(); auto s = sst_file_reader->VerifyChecksum();
if (!s.ok()) { if (!s.ok()) {
rocksdb::RocksDBExceptionJni::ThrowNew(env, s); rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
@ -99,12 +99,12 @@ void Java_org_rocksdb_SstFileReader_verifyChecksum(JNIEnv *env,
* Signature: (J)J * Signature: (J)J
*/ */
jobject Java_org_rocksdb_SstFileReader_getTableProperties(JNIEnv *env, jobject Java_org_rocksdb_SstFileReader_getTableProperties(JNIEnv *env,
jobject /*jobj*/, jobject /*jobj*/,
jlong jhandle) { jlong jhandle) {
auto* sst_file_reader = reinterpret_cast<rocksdb::SstFileReader*>(jhandle); auto *sst_file_reader = reinterpret_cast<rocksdb::SstFileReader *>(jhandle);
std::shared_ptr<const rocksdb::TableProperties> tp = sst_file_reader->GetTableProperties(); std::shared_ptr<const rocksdb::TableProperties> tp =
jobject jtable_properties = rocksdb::TablePropertiesJni::fromCppTableProperties( sst_file_reader->GetTableProperties();
env, *(tp.get())); jobject jtable_properties =
rocksdb::TablePropertiesJni::fromCppTableProperties(env, *(tp.get()));
return jtable_properties; return jtable_properties;
} }

25
java/src/main/java/org/rocksdb/SstFileReader.java
Unescape View File

@ -31,19 +31,18 @@ public class SstFileReader extends RocksObject {
* @return instance of iterator object. * @return instance of iterator object.
*/ */
public SstFileReaderIterator newIterator(final ReadOptions readOptions) { public SstFileReaderIterator newIterator(final ReadOptions readOptions) {
assert(isOwningHandle()); assert (isOwningHandle());
long iter = newIterator(nativeHandle_, long iter = newIterator(nativeHandle_, readOptions.nativeHandle_);
readOptions.nativeHandle_);
return new SstFileReaderIterator(this, iter); return new SstFileReaderIterator(this, iter);
} }
/** /**
* Prepare SstFileReader to read a file. * Prepare SstFileReader to read a file.
* *
* @param filePath the location of file * @param filePath the location of file
* *
* @throws RocksDBException thrown if error happens in underlying * @throws RocksDBException thrown if error happens in underlying
* native library. * native library.
*/ */
public void open(final String filePath) throws RocksDBException { public void open(final String filePath) throws RocksDBException {
open(nativeHandle_, filePath); open(nativeHandle_, filePath);
@ -68,14 +67,10 @@ public class SstFileReader extends RocksObject {
return getTableProperties(nativeHandle_); return getTableProperties(nativeHandle_);
} }
@Override protected final native void disposeInternal(final long handle); @Override protected final native void disposeInternal(final long handle);
private native long newIterator(final long handle, private native long newIterator(final long handle, final long readOptionsHandle);
final long readOptionsHandle);
private native void open(final long handle, final String filePath) private native void open(final long handle, final String filePath) throws RocksDBException;
throws RocksDBException;
private native static long newSstFileReader(final long optionsHandle); private native static long newSstFileReader(final long optionsHandle);
private native void verifyChecksum(final long handle) throws RocksDBException; private native void verifyChecksum(final long handle) throws RocksDBException;

4
java/src/main/java/org/rocksdb/SstFileReaderIterator.java
Unescape View File

@ -33,7 +33,7 @@ public class SstFileReaderIterator extends AbstractRocksIterator<SstFileReader>
* @return key for the current entry. * @return key for the current entry.
*/ */
public byte[] key() { public byte[] key() {
assert(isOwningHandle()); assert (isOwningHandle());
return key0(nativeHandle_); return key0(nativeHandle_);
} }
@ -46,7 +46,7 @@ public class SstFileReaderIterator extends AbstractRocksIterator<SstFileReader>
* @return value for the current entry. * @return value for the current entry.
*/ */
public byte[] value() { public byte[] value() {
assert(isOwningHandle()); assert (isOwningHandle());
return value0(nativeHandle_); return value0(nativeHandle_);
} }

18
java/src/test/java/org/rocksdb/SstFileReaderTest.java
Unescape View File

@ -48,9 +48,10 @@ public class SstFileReaderTest {
@Rule public TemporaryFolder parentFolder = new TemporaryFolder(); @Rule public TemporaryFolder parentFolder = new TemporaryFolder();
enum OpType { PUT, PUT_BYTES, MERGE, MERGE_BYTES, DELETE, DELETE_BYTES} enum OpType { PUT, PUT_BYTES, MERGE, MERGE_BYTES, DELETE, DELETE_BYTES }
private File newSstFile(final List<KeyValueWithOp> keyValues) throws IOException, RocksDBException { private File newSstFile(final List<KeyValueWithOp> keyValues)
throws IOException, RocksDBException {
final EnvOptions envOptions = new EnvOptions(); final EnvOptions envOptions = new EnvOptions();
final StringAppendOperator stringAppendOperator = new StringAppendOperator(); final StringAppendOperator stringAppendOperator = new StringAppendOperator();
final Options options = new Options().setMergeOperator(stringAppendOperator); final Options options = new Options().setMergeOperator(stringAppendOperator);
@ -105,15 +106,11 @@ public class SstFileReaderTest {
final List<KeyValueWithOp> keyValues = new ArrayList<>(); final List<KeyValueWithOp> keyValues = new ArrayList<>();
keyValues.add(new KeyValueWithOp("key1", "value1", OpType.PUT)); keyValues.add(new KeyValueWithOp("key1", "value1", OpType.PUT));
final File sstFile = newSstFile(keyValues); final File sstFile = newSstFile(keyValues);
try(final StringAppendOperator stringAppendOperator = try (final StringAppendOperator stringAppendOperator = new StringAppendOperator();
new StringAppendOperator(); final Options options =
final Options options = new Options() new Options().setCreateIfMissing(true).setMergeOperator(stringAppendOperator);
.setCreateIfMissing(true) final SstFileReader reader = new SstFileReader(options)) {
.setMergeOperator(stringAppendOperator);
final SstFileReader reader = new SstFileReader(options)
) {
// Open the sst file and iterator // Open the sst file and iterator
reader.open(sstFile.getAbsolutePath()); reader.open(sstFile.getAbsolutePath());
final ReadOptions readOptions = new ReadOptions(); final ReadOptions readOptions = new ReadOptions();
@ -133,5 +130,4 @@ public class SstFileReaderTest {
assertThat(iterator.value()).isEqualTo("value1".getBytes()); assertThat(iterator.value()).isEqualTo("value1".getBytes());
} }
} }
} }

3
options/options_settable_test.cc
Unescape View File

@ -356,8 +356,7 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
{offset_of(&ColumnFamilyOptions::snap_refresh_nanos), sizeof(uint64_t)}, {offset_of(&ColumnFamilyOptions::snap_refresh_nanos), sizeof(uint64_t)},
{offset_of(&ColumnFamilyOptions::table_factory), {offset_of(&ColumnFamilyOptions::table_factory),
sizeof(std::shared_ptr<TableFactory>)}, sizeof(std::shared_ptr<TableFactory>)},
{offset_of(&ColumnFamilyOptions::cf_paths), {offset_of(&ColumnFamilyOptions::cf_paths), sizeof(std::vector<DbPath>)},
sizeof(std::vector<DbPath>)},
{offset_of(&ColumnFamilyOptions::compaction_thread_limiter), {offset_of(&ColumnFamilyOptions::compaction_thread_limiter),
sizeof(std::shared_ptr<ConcurrentTaskLimiter>)}, sizeof(std::shared_ptr<ConcurrentTaskLimiter>)},
}; };

6
tools/block_cache_analyzer/block_cache_trace_analyzer.cc
Unescape View File

@ -1830,8 +1830,10 @@ void BlockCacheTraceAnalyzer::PrintDataBlockAccessStats() const {
hist_naccess_per_key.Add(caller_access.second); hist_naccess_per_key.Add(caller_access.second);
} }
} }
uint64_t avg_accesses = static_cast<uint64_t>(hist_naccess_per_key.Average()); uint64_t avg_accesses =
uint64_t stdev_accesses = static_cast<uint64_t>(hist_naccess_per_key.StandardDeviation()); static_cast<uint64_t>(hist_naccess_per_key.Average());
uint64_t stdev_accesses =
static_cast<uint64_t>(hist_naccess_per_key.StandardDeviation());
avg_naccesses_per_key_in_a_data_block.Add(avg_accesses); avg_naccesses_per_key_in_a_data_block.Add(avg_accesses);
cf_avg_naccesses_per_key_in_a_data_block[cf_name].Add(avg_accesses); cf_avg_naccesses_per_key_in_a_data_block[cf_name].Add(avg_accesses);
stdev_naccesses_per_key_in_a_data_block.Add(stdev_accesses); stdev_naccesses_per_key_in_a_data_block.Add(stdev_accesses);

5
tools/db_stress.cc
Unescape View File

@ -353,8 +353,9 @@ DEFINE_int32(bloom_bits, 10, "Bloom filter bits per key. "
DEFINE_bool(use_block_based_filter, false, "use block based filter" DEFINE_bool(use_block_based_filter, false, "use block based filter"
"instead of full filter for block based table"); "instead of full filter for block based table");
DEFINE_bool(partition_filters, false, "use partitioned filters " DEFINE_bool(partition_filters, false,
"for block-based table"); "use partitioned filters "
"for block-based table");
DEFINE_int32( DEFINE_int32(
index_type, index_type,

5
tools/sst_dump_tool.cc
Unescape View File

@ -405,8 +405,9 @@ Status SstFileDumper::ReadTableProperties(
namespace { namespace {
void print_help() { void print_help() {
fprintf(stderr, fprintf(
R"(sst_dump --file=<data_dir_OR_sst_file> [--command=check|scan|raw|recompress] stderr,
R"(sst_dump --file=<data_dir_OR_sst_file> [--command=check|scan|raw|recompress]
--file=<data_dir_OR_sst_file> --file=<data_dir_OR_sst_file>
Path to SST file or directory containing SST files Path to SST file or directory containing SST files

30
util/bloom_impl.h
Unescape View File

@ -26,13 +26,13 @@ namespace rocksdb {
// See e.g. RocksDB DynamicBloom. // See e.g. RocksDB DynamicBloom.
// //
class LegacyNoLocalityBloomImpl { class LegacyNoLocalityBloomImpl {
public: public:
static inline void AddHash(uint32_t h, uint32_t total_bits, static inline void AddHash(uint32_t h, uint32_t total_bits, int num_probes,
int num_probes, char *data) { char *data) {
const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
for (int i = 0; i < num_probes; i++) { for (int i = 0; i < num_probes; i++) {
const uint32_t bitpos = h % total_bits; const uint32_t bitpos = h % total_bits;
data[bitpos/8] |= (1 << (bitpos % 8)); data[bitpos / 8] |= (1 << (bitpos % 8));
h += delta; h += delta;
} }
} }
@ -42,7 +42,7 @@ public:
const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
for (int i = 0; i < num_probes; i++) { for (int i = 0; i < num_probes; i++) {
const uint32_t bitpos = h % total_bits; const uint32_t bitpos = h % total_bits;
if ((data[bitpos/8] & (1 << (bitpos % 8))) == 0) { if ((data[bitpos / 8] & (1 << (bitpos % 8))) == 0) {
return false; return false;
} }
h += delta; h += delta;
@ -51,7 +51,6 @@ public:
} }
}; };
// A legacy Bloom filter implementation with probes local to a single // A legacy Bloom filter implementation with probes local to a single
// cache line (fast). Because SST files might be transported between // cache line (fast). Because SST files might be transported between
// platforms, the cache line size is a parameter rather than hard coded. // platforms, the cache line size is a parameter rather than hard coded.
@ -72,15 +71,15 @@ public:
// //
template <bool ExtraRotates> template <bool ExtraRotates>
class LegacyLocalityBloomImpl { class LegacyLocalityBloomImpl {
private: private:
static inline uint32_t GetLine(uint32_t h, uint32_t num_lines) { static inline uint32_t GetLine(uint32_t h, uint32_t num_lines) {
uint32_t offset_h = ExtraRotates ? (h >> 11) | (h << 21) : h; uint32_t offset_h = ExtraRotates ? (h >> 11) | (h << 21) : h;
return offset_h % num_lines; return offset_h % num_lines;
} }
public:
static inline void AddHash(uint32_t h, uint32_t num_lines, public:
int num_probes, char *data, static inline void AddHash(uint32_t h, uint32_t num_lines, int num_probes,
int log2_cache_line_bytes) { char *data, int log2_cache_line_bytes) {
const int log2_cache_line_bits = log2_cache_line_bytes + 3; const int log2_cache_line_bits = log2_cache_line_bytes + 3;
char *data_at_offset = char *data_at_offset =
@ -99,12 +98,12 @@ public:
static inline void PrepareHashMayMatch(uint32_t h, uint32_t num_lines, static inline void PrepareHashMayMatch(uint32_t h, uint32_t num_lines,
const char *data, const char *data,
uint32_t /*out*/*byte_offset, uint32_t /*out*/ *byte_offset,
int log2_cache_line_bytes) { int log2_cache_line_bytes) {
uint32_t b = GetLine(h, num_lines) << log2_cache_line_bytes; uint32_t b = GetLine(h, num_lines) << log2_cache_line_bytes;
PREFETCH(data + b, 0 /* rw */, 1 /* locality */); PREFETCH(data + b, 0 /* rw */, 1 /* locality */);
PREFETCH(data + b + ((1 << log2_cache_line_bytes) - 1), PREFETCH(data + b + ((1 << log2_cache_line_bytes) - 1), 0 /* rw */,
0 /* rw */, 1 /* locality */); 1 /* locality */);
*byte_offset = b; *byte_offset = b;
} }
@ -112,8 +111,7 @@ public:
int num_probes, const char *data, int num_probes, const char *data,
int log2_cache_line_bytes) { int log2_cache_line_bytes) {
uint32_t b = GetLine(h, num_lines) << log2_cache_line_bytes; uint32_t b = GetLine(h, num_lines) << log2_cache_line_bytes;
return HashMayMatchPrepared(h, num_probes, return HashMayMatchPrepared(h, num_probes, data + b, log2_cache_line_bytes);
data + b, log2_cache_line_bytes);
} }
static inline bool HashMayMatchPrepared(uint32_t h, int num_probes, static inline bool HashMayMatchPrepared(uint32_t h, int num_probes,

4
util/concurrent_task_limiter_impl.h
Unescape View File

@ -24,8 +24,8 @@ class ConcurrentTaskLimiterImpl : public ConcurrentTaskLimiter {
int32_t max_outstanding_task); int32_t max_outstanding_task);
// No copying allowed // No copying allowed
ConcurrentTaskLimiterImpl(const ConcurrentTaskLimiterImpl&) = delete; ConcurrentTaskLimiterImpl(const ConcurrentTaskLimiterImpl&) = delete;
ConcurrentTaskLimiterImpl& operator=( ConcurrentTaskLimiterImpl& operator=(const ConcurrentTaskLimiterImpl&) =
const ConcurrentTaskLimiterImpl&) = delete; delete;
virtual ~ConcurrentTaskLimiterImpl(); virtual ~ConcurrentTaskLimiterImpl();

26
util/crc32c_arm64.h
Unescape View File

@ -17,17 +17,21 @@
#define crc32c_u16(crc, v) __crc32ch(crc, v) #define crc32c_u16(crc, v) __crc32ch(crc, v)
#define crc32c_u32(crc, v) __crc32cw(crc, v) #define crc32c_u32(crc, v) __crc32cw(crc, v)
#define crc32c_u64(crc, v) __crc32cd(crc, v) #define crc32c_u64(crc, v) __crc32cd(crc, v)
#define PREF4X64L1(buffer,PREF_OFFSET, ITR) \ #define PREF4X64L1(buffer, PREF_OFFSET, ITR) \
__asm__("PRFM PLDL1KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 0)*64));\ __asm__("PRFM PLDL1KEEP, [%x[v],%[c]]" ::[v] "r"(buffer), \
__asm__("PRFM PLDL1KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 1)*64));\ [c] "I"((PREF_OFFSET) + ((ITR) + 0) * 64)); \
__asm__("PRFM PLDL1KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 2)*64));\ __asm__("PRFM PLDL1KEEP, [%x[v],%[c]]" ::[v] "r"(buffer), \
__asm__("PRFM PLDL1KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 3)*64)); [c] "I"((PREF_OFFSET) + ((ITR) + 1) * 64)); \
__asm__("PRFM PLDL1KEEP, [%x[v],%[c]]" ::[v] "r"(buffer), \
#define PREF1KL1(buffer,PREF_OFFSET) \ [c] "I"((PREF_OFFSET) + ((ITR) + 2) * 64)); \
PREF4X64L1(buffer,(PREF_OFFSET), 0) \ __asm__("PRFM PLDL1KEEP, [%x[v],%[c]]" ::[v] "r"(buffer), \
PREF4X64L1(buffer,(PREF_OFFSET), 4) \ [c] "I"((PREF_OFFSET) + ((ITR) + 3) * 64));
PREF4X64L1(buffer,(PREF_OFFSET), 8) \
PREF4X64L1(buffer,(PREF_OFFSET), 12) #define PREF1KL1(buffer, PREF_OFFSET) \
PREF4X64L1(buffer, (PREF_OFFSET), 0) \
PREF4X64L1(buffer, (PREF_OFFSET), 4) \
PREF4X64L1(buffer, (PREF_OFFSET), 8) \
PREF4X64L1(buffer, (PREF_OFFSET), 12)
extern uint32_t crc32c_arm64(uint32_t crc, unsigned char const *data, unsigned len); extern uint32_t crc32c_arm64(uint32_t crc, unsigned char const *data, unsigned len);
extern uint32_t crc32c_runtime_check(void); extern uint32_t crc32c_runtime_check(void);

2
util/dynamic_bloom.cc
Unescape View File

@ -42,7 +42,7 @@ DynamicBloom::DynamicBloom(Allocator* allocator, uint32_t total_bits,
uint32_t block_bits = block_bytes * 8; uint32_t block_bits = block_bytes * 8;
uint32_t blocks = (total_bits + block_bits - 1) / block_bits; uint32_t blocks = (total_bits + block_bits - 1) / block_bits;
uint32_t sz = blocks * block_bytes; uint32_t sz = blocks * block_bytes;
kLen = sz / /*bytes/u64*/8; kLen = sz / /*bytes/u64*/ 8;
assert(kLen > 0); assert(kLen > 0);
#ifndef NDEBUG #ifndef NDEBUG
for (uint32_t i = 0; i < kNumDoubleProbes; ++i) { for (uint32_t i = 0; i < kNumDoubleProbes; ++i) {

66
util/dynamic_bloom_test.cc
Unescape View File

@ -134,41 +134,41 @@ TEST_F(DynamicBloomTest, VaryingLengths) {
// NB: FP rate impact of 32-bit hash is noticeable starting around 10M keys. // NB: FP rate impact of 32-bit hash is noticeable starting around 10M keys.
// But that effect is hidden if using sequential keys (unique hashes). // But that effect is hidden if using sequential keys (unique hashes).
for (bool nonseq : {false, true}) { for (bool nonseq : {false, true}) {
const uint32_t max_num = FLAGS_enable_perf ? 40000000 : 400000; const uint32_t max_num = FLAGS_enable_perf ? 40000000 : 400000;
for (uint32_t num = 1; num <= max_num; num = NextNum(num)) { for (uint32_t num = 1; num <= max_num; num = NextNum(num)) {
uint32_t bloom_bits = 0; uint32_t bloom_bits = 0;
Arena arena; Arena arena;
bloom_bits = num * FLAGS_bits_per_key; bloom_bits = num * FLAGS_bits_per_key;
DynamicBloom bloom(&arena, bloom_bits, num_probes); DynamicBloom bloom(&arena, bloom_bits, num_probes);
for (uint64_t i = 0; i < num; i++) { for (uint64_t i = 0; i < num; i++) {
bloom.Add(km.Key(i, nonseq)); bloom.Add(km.Key(i, nonseq));
ASSERT_TRUE(bloom.MayContain(km.Key(i, nonseq))); ASSERT_TRUE(bloom.MayContain(km.Key(i, nonseq)));
} }
// All added keys must match // All added keys must match
for (uint64_t i = 0; i < num; i++) { for (uint64_t i = 0; i < num; i++) {
ASSERT_TRUE(bloom.MayContain(km.Key(i, nonseq))); ASSERT_TRUE(bloom.MayContain(km.Key(i, nonseq)));
} }
// Check false positive rate // Check false positive rate
int result = 0; int result = 0;
for (uint64_t i = 0; i < 30000; i++) { for (uint64_t i = 0; i < 30000; i++) {
if (bloom.MayContain(km.Key(i + 1000000000, nonseq))) { if (bloom.MayContain(km.Key(i + 1000000000, nonseq))) {
result++; result++;
}
} }
} double rate = result / 30000.0;
double rate = result / 30000.0;
fprintf(stderr, fprintf(stderr,
"False positives (%s keys): " "False positives (%s keys): "
"%5.2f%% @ num = %6u, bloom_bits = %6u\n", "%5.2f%% @ num = %6u, bloom_bits = %6u\n",
nonseq ? "nonseq" : "seq", rate * 100.0, num, bloom_bits); nonseq ? "nonseq" : "seq", rate * 100.0, num, bloom_bits);
if (rate > 0.0125) if (rate > 0.0125)
mediocre_filters++; // Allowed, but not too often mediocre_filters++; // Allowed, but not too often
else else
good_filters++; good_filters++;
} }
} }
fprintf(stderr, "Filters: %d good, %d mediocre\n", good_filters, fprintf(stderr, "Filters: %d good, %d mediocre\n", good_filters,
@ -263,8 +263,7 @@ TEST_F(DynamicBloomTest, concurrent_with_perf) {
StopWatchNano timer(Env::Default()); StopWatchNano timer(Env::Default());
timer.Start(); timer.Start();
for (uint64_t i = 1 + t; i <= num_keys; i += num_threads) { for (uint64_t i = 1 + t; i <= num_keys; i += num_threads) {
bool f = bool f = std_bloom.MayContain(km.Seq(i));
std_bloom.MayContain(km.Seq(i));
ASSERT_TRUE(f); ASSERT_TRUE(f);
} }
elapsed += timer.ElapsedNanos(); elapsed += timer.ElapsedNanos();
@ -289,8 +288,7 @@ TEST_F(DynamicBloomTest, concurrent_with_perf) {
timer.Start(); timer.Start();
for (uint64_t i = num_keys + 1 + t; i <= 2 * num_keys; for (uint64_t i = num_keys + 1 + t; i <= 2 * num_keys;
i += num_threads) { i += num_threads) {
bool f = bool f = std_bloom.MayContain(km.Seq(i));
std_bloom.MayContain(km.Seq(i));
if (f) { if (f) {
++false_positives; ++false_positives;
} }

12
util/mutexlock.h
Unescape View File

@ -32,8 +32,8 @@ class MutexLock {
this->mu_->Lock(); this->mu_->Lock();
} }
// No copying allowed // No copying allowed
MutexLock(const MutexLock&) = delete; MutexLock(const MutexLock &) = delete;
void operator=(const MutexLock&) = delete; void operator=(const MutexLock &) = delete;
~MutexLock() { this->mu_->Unlock(); } ~MutexLock() { this->mu_->Unlock(); }
@ -52,8 +52,8 @@ class ReadLock {
this->mu_->ReadLock(); this->mu_->ReadLock();
} }
// No copying allowed // No copying allowed
ReadLock(const ReadLock&) = delete; ReadLock(const ReadLock &) = delete;
void operator=(const ReadLock&) = delete; void operator=(const ReadLock &) = delete;
~ReadLock() { this->mu_->ReadUnlock(); } ~ReadLock() { this->mu_->ReadUnlock(); }
@ -88,8 +88,8 @@ class WriteLock {
this->mu_->WriteLock(); this->mu_->WriteLock();
} }
// No copying allowed // No copying allowed
WriteLock(const WriteLock&) = delete; WriteLock(const WriteLock &) = delete;
void operator=(const WriteLock&) = delete; void operator=(const WriteLock &) = delete;
~WriteLock() { this->mu_->WriteUnlock(); } ~WriteLock() { this->mu_->WriteUnlock(); }

2
utilities/transactions/write_unprepared_txn_db.cc
Unescape View File

@ -5,8 +5,8 @@
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#include "db/arena_wrapped_db_iter.h"
#include "utilities/transactions/write_unprepared_txn_db.h" #include "utilities/transactions/write_unprepared_txn_db.h"
#include "db/arena_wrapped_db_iter.h"
#include "rocksdb/utilities/transaction_db.h" #include "rocksdb/utilities/transaction_db.h"
#include "util/cast_util.h" #include "util/cast_util.h"

Write Preview
Loading…
Cancel
Save