Ran clang-format on db/ directory (#10910)

Summary:
Ran `find ./db/ -type f | xargs clang-format -i`. Excluded minor changes it tried to make on db/db_impl/. Everything else it changed was directly under db/ directory. Included minor manual touchups mentioned in PR commit history.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10910

Reviewed By: riversand963

Differential Revision: D40880683

Pulled By: ajkr

fbshipit-source-id: cfe26cda05b3fb9a72e3cb82c286e21d8c5c4174
main
Andrew Kryczka 2 years ago committed by Facebook GitHub Bot
parent ff9ad2c39b
commit 5cf6ab6f31
  1. 1
      db/arena_wrapped_db_iter.cc
  2. 2
      db/arena_wrapped_db_iter.h
  3. 3
      db/builder.cc
  4. 1236
      db/c.cc
  5. 337
      db/c_test.c
  6. 31
      db/column_family.cc
  7. 14
      db/column_family.h
  8. 45
      db/column_family_test.cc
  9. 13
      db/compact_files_test.cc
  10. 5
      db/comparator_db_test.cc
  11. 8
      db/convenience.cc
  12. 24
      db/corruption_test.cc
  13. 22
      db/cuckoo_table_db_test.cc
  14. 4
      db/db_basic_test.cc
  15. 4
      db/db_block_cache_test.cc
  16. 30
      db/db_bloom_filter_test.cc
  17. 124
      db/db_compaction_test.cc
  18. 4
      db/db_dynamic_level_test.cc
  19. 24
      db/db_encryption_test.cc
  20. 3
      db/db_filesnapshot.cc
  21. 6
      db/db_flush_test.cc
  22. 1
      db/db_info_dumper.cc
  23. 13
      db/db_iter_stress_test.cc
  24. 18
      db/db_iter_test.cc
  25. 28
      db/db_iterator_test.cc
  26. 16
      db/db_log_iter_test.cc
  27. 8
      db/db_logical_block_size_cache_test.cc
  28. 2
      db/db_merge_operand_test.cc
  29. 6
      db/db_merge_operator_test.cc
  30. 8
      db/db_options_test.cc
  31. 11
      db/db_properties_test.cc
  32. 12
      db/db_range_del_test.cc
  33. 2
      db/db_secondary_test.cc
  34. 14
      db/db_statistics_test.cc
  35. 41
      db/db_table_properties_test.cc
  36. 8
      db/db_tailing_iter_test.cc
  37. 10
      db/db_test.cc
  38. 341
      db/db_test2.cc
  39. 134
      db/db_test_util.cc
  40. 4
      db/db_test_util.h
  41. 70
      db/db_universal_compaction_test.cc
  42. 83
      db/db_write_test.cc
  43. 69
      db/dbformat_test.cc
  44. 21
      db/deletefile_test.cc
  45. 12
      db/error_handler.cc
  46. 187
      db/error_handler.h
  47. 2
      db/event_helpers.cc
  48. 6
      db/event_helpers.h
  49. 33
      db/external_sst_file_ingestion_job.cc
  50. 4
      db/external_sst_file_test.cc
  51. 3
      db/fault_injection_test.cc
  52. 2
      db/file_indexer.cc
  53. 3
      db/file_indexer.h
  54. 6
      db/file_indexer_test.cc
  55. 56
      db/filename_test.cc
  56. 20
      db/flush_job.cc
  57. 4
      db/flush_job_test.cc
  58. 36
      db/forward_iterator.cc
  59. 13
      db/forward_iterator.h
  60. 6
      db/forward_iterator_bench.cc
  61. 4
      db/import_column_family_job.cc
  62. 21
      db/internal_stats.h
  63. 11
      db/job_context.h
  64. 44
      db/listener_test.cc
  65. 18
      db/log_reader.cc
  66. 10
      db/log_reader.h
  67. 18
      db/log_test.cc
  68. 2
      db/log_writer.h
  69. 1
      db/logs_with_prep_tracker.h
  70. 1
      db/lookup_key.h
  71. 3
      db/malloc_stats.cc
  72. 8
      db/manual_compaction_test.cc
  73. 7
      db/memtable.cc
  74. 10
      db/memtable.h
  75. 2
      db/memtable_list_test.cc
  76. 1
      db/merge_context.h
  77. 2
      db/merge_helper.h
  78. 11
      db/merge_operator.cc
  79. 7
      db/obsolete_files_test.cc
  80. 2
      db/options_file_test.cc
  81. 98
      db/perf_context_test.cc
  82. 587
      db/plain_table_db_test.cc
  83. 49
      db/prefix_test.cc
  84. 2
      db/range_del_aggregator.cc
  85. 2
      db/range_del_aggregator_test.cc
  86. 15
      db/repair.cc
  87. 7
      db/repair_test.cc
  88. 9
      db/snapshot_impl.cc
  89. 14
      db/snapshot_impl.h
  90. 7
      db/table_cache.cc
  91. 13
      db/table_properties_collector.cc
  92. 44
      db/table_properties_collector_test.cc
  93. 24
      db/transaction_log_impl.cc
  94. 14
      db/transaction_log_impl.h
  95. 2
      db/trim_history_scheduler.h
  96. 6
      db/version_builder_test.cc
  97. 7
      db/version_edit.cc
  98. 20
      db/version_edit.h
  99. 164
      db/version_set.cc
  100. 18
      db/version_set.h
  101. Some files were not shown because too many files have changed in this diff Show More

@ -8,6 +8,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/arena_wrapped_db_iter.h" #include "db/arena_wrapped_db_iter.h"
#include "memory/arena.h" #include "memory/arena.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/iterator.h" #include "rocksdb/iterator.h"

@ -9,7 +9,9 @@
#pragma once #pragma once
#include <stdint.h> #include <stdint.h>
#include <string> #include <string>
#include "db/db_impl/db_impl.h" #include "db/db_impl/db_impl.h"
#include "db/db_iter.h" #include "db/db_iter.h"
#include "db/range_del_aggregator.h" #include "db/range_del_aggregator.h"

@ -281,7 +281,8 @@ Status BuildTable(
meta->fd.file_size = file_size; meta->fd.file_size = file_size;
meta->marked_for_compaction = builder->NeedCompact(); meta->marked_for_compaction = builder->NeedCompact();
assert(meta->fd.GetFileSize() > 0); assert(meta->fd.GetFileSize() > 0);
tp = builder->GetTableProperties(); // refresh now that builder is finished tp = builder
->GetTableProperties(); // refresh now that builder is finished
if (memtable_payload_bytes != nullptr && if (memtable_payload_bytes != nullptr &&
memtable_garbage_bytes != nullptr) { memtable_garbage_bytes != nullptr) {
const CompactionIterationStats& ci_stats = c_iter.iter_stats(); const CompactionIterationStats& ci_stats = c_iter.iter_stats();

1236
db/c.cc

File diff suppressed because it is too large Load Diff

@ -49,32 +49,32 @@ static void StartPhase(const char* name) {
} }
#ifdef _MSC_VER #ifdef _MSC_VER
#pragma warning(push) #pragma warning(push)
#pragma warning (disable: 4996) // getenv security warning #pragma warning(disable : 4996) // getenv security warning
#endif #endif
static const char* GetTempDir(void) { static const char* GetTempDir(void) {
const char* ret = getenv("TEST_TMPDIR"); const char* ret = getenv("TEST_TMPDIR");
if (ret == NULL || ret[0] == '\0') if (ret == NULL || ret[0] == '\0')
#ifdef OS_WIN #ifdef OS_WIN
ret = getenv("TEMP"); ret = getenv("TEMP");
#else #else
ret = "/tmp"; ret = "/tmp";
#endif #endif
return ret; return ret;
} }
#ifdef _MSC_VER #ifdef _MSC_VER
#pragma warning(pop) #pragma warning(pop)
#endif #endif
#define CheckNoError(err) \ #define CheckNoError(err) \
if ((err) != NULL) { \ if ((err) != NULL) { \
fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \ fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \
abort(); \ abort(); \
} }
#define CheckCondition(cond) \ #define CheckCondition(cond) \
if (!(cond)) { \ if (!(cond)) { \
fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \ fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \
abort(); \ abort(); \
} }
static void CheckEqual(const char* expected, const char* v, size_t n) { static void CheckEqual(const char* expected, const char* v, size_t n) {
@ -98,21 +98,15 @@ static void Free(char** ptr) {
} }
} }
static void CheckValue( static void CheckValue(char* err, const char* expected, char** actual,
char* err, size_t actual_length) {
const char* expected,
char** actual,
size_t actual_length) {
CheckNoError(err); CheckNoError(err);
CheckEqual(expected, *actual, actual_length); CheckEqual(expected, *actual, actual_length);
Free(actual); Free(actual);
} }
static void CheckGet( static void CheckGet(rocksdb_t* db, const rocksdb_readoptions_t* options,
rocksdb_t* db, const char* key, const char* expected) {
const rocksdb_readoptions_t* options,
const char* key,
const char* expected) {
char* err = NULL; char* err = NULL;
size_t val_len; size_t val_len;
char* val; char* val;
@ -122,12 +116,9 @@ static void CheckGet(
Free(&val); Free(&val);
} }
static void CheckGetCF( static void CheckGetCF(rocksdb_t* db, const rocksdb_readoptions_t* options,
rocksdb_t* db, rocksdb_column_family_handle_t* handle, const char* key,
const rocksdb_readoptions_t* options, const char* expected) {
rocksdb_column_family_handle_t* handle,
const char* key,
const char* expected) {
char* err = NULL; char* err = NULL;
size_t val_len; size_t val_len;
char* val; char* val;
@ -174,8 +165,8 @@ static void CheckMultiGetValues(size_t num_keys, char** values,
} }
} }
static void CheckIter(rocksdb_iterator_t* iter, static void CheckIter(rocksdb_iterator_t* iter, const char* key,
const char* key, const char* val) { const char* val) {
size_t len; size_t len;
const char* str; const char* str;
str = rocksdb_iter_key(iter, &len); str = rocksdb_iter_key(iter, &len);
@ -185,10 +176,9 @@ static void CheckIter(rocksdb_iterator_t* iter,
} }
// Callback from rocksdb_writebatch_iterate() // Callback from rocksdb_writebatch_iterate()
static void CheckPut(void* ptr, static void CheckPut(void* ptr, const char* k, size_t klen, const char* v,
const char* k, size_t klen, size_t vlen) {
const char* v, size_t vlen) { int* state = (int*)ptr;
int* state = (int*) ptr;
CheckCondition(*state < 2); CheckCondition(*state < 2);
switch (*state) { switch (*state) {
case 0: case 0:
@ -205,7 +195,7 @@ static void CheckPut(void* ptr,
// Callback from rocksdb_writebatch_iterate() // Callback from rocksdb_writebatch_iterate()
static void CheckDel(void* ptr, const char* k, size_t klen) { static void CheckDel(void* ptr, const char* k, size_t klen) {
int* state = (int*) ptr; int* state = (int*)ptr;
CheckCondition(*state == 2); CheckCondition(*state == 2);
CheckEqual("bar", k, klen); CheckEqual("bar", k, klen);
(*state)++; (*state)++;
@ -213,14 +203,16 @@ static void CheckDel(void* ptr, const char* k, size_t klen) {
static void CmpDestroy(void* arg) { (void)arg; } static void CmpDestroy(void* arg) { (void)arg; }
static int CmpCompare(void* arg, const char* a, size_t alen, static int CmpCompare(void* arg, const char* a, size_t alen, const char* b,
const char* b, size_t blen) { size_t blen) {
(void)arg; (void)arg;
size_t n = (alen < blen) ? alen : blen; size_t n = (alen < blen) ? alen : blen;
int r = memcmp(a, b, n); int r = memcmp(a, b, n);
if (r == 0) { if (r == 0) {
if (alen < blen) r = -1; if (alen < blen)
else if (alen > blen) r = +1; r = -1;
else if (alen > blen)
r = +1;
} }
return r; return r;
} }
@ -405,11 +397,9 @@ static const char* MergeOperatorName(void* arg) {
return "TestMergeOperator"; return "TestMergeOperator";
} }
static char* MergeOperatorFullMerge( static char* MergeOperatorFullMerge(
void* arg, void* arg, const char* key, size_t key_length, const char* existing_value,
const char* key, size_t key_length, size_t existing_value_length, const char* const* operands_list,
const char* existing_value, size_t existing_value_length, const size_t* operands_list_length, int num_operands,
const char* const* operands_list, const size_t* operands_list_length,
int num_operands,
unsigned char* success, size_t* new_value_length) { unsigned char* success, size_t* new_value_length) {
(void)arg; (void)arg;
(void)key; (void)key;
@ -425,12 +415,12 @@ static char* MergeOperatorFullMerge(
memcpy(result, "fake", 4); memcpy(result, "fake", 4);
return result; return result;
} }
static char* MergeOperatorPartialMerge( static char* MergeOperatorPartialMerge(void* arg, const char* key,
void* arg, size_t key_length,
const char* key, size_t key_length, const char* const* operands_list,
const char* const* operands_list, const size_t* operands_list_length, const size_t* operands_list_length,
int num_operands, int num_operands, unsigned char* success,
unsigned char* success, size_t* new_value_length) { size_t* new_value_length) {
(void)arg; (void)arg;
(void)key; (void)key;
(void)key_length; (void)key_length;
@ -444,18 +434,16 @@ static char* MergeOperatorPartialMerge(
return result; return result;
} }
static void CheckTxnGet( static void CheckTxnGet(rocksdb_transaction_t* txn,
rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, const char* key,
const rocksdb_readoptions_t* options, const char* expected) {
const char* key, char* err = NULL;
const char* expected) { size_t val_len;
char* err = NULL; char* val;
size_t val_len; val = rocksdb_transaction_get(txn, options, key, strlen(key), &val_len, &err);
char* val; CheckNoError(err);
val = rocksdb_transaction_get(txn, options, key, strlen(key), &val_len, &err); CheckEqual(expected, val, val_len);
CheckNoError(err); Free(&val);
CheckEqual(expected, val, val_len);
Free(&val);
} }
static void CheckTxnGetCF(rocksdb_transaction_t* txn, static void CheckTxnGetCF(rocksdb_transaction_t* txn,
@ -502,11 +490,9 @@ static void CheckTxnPinGetCF(rocksdb_transaction_t* txn,
rocksdb_pinnableslice_destroy(p); rocksdb_pinnableslice_destroy(p);
} }
static void CheckTxnDBGet( static void CheckTxnDBGet(rocksdb_transactiondb_t* txn_db,
rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, const char* key,
const rocksdb_readoptions_t* options, const char* expected) {
const char* key,
const char* expected) {
char* err = NULL; char* err = NULL;
size_t val_len; size_t val_len;
char* val; char* val;
@ -632,7 +618,7 @@ int main(int argc, char** argv) {
rocksdb_t* db; rocksdb_t* db;
rocksdb_comparator_t* cmp; rocksdb_comparator_t* cmp;
rocksdb_cache_t* cache; rocksdb_cache_t* cache;
rocksdb_dbpath_t *dbpath; rocksdb_dbpath_t* dbpath;
rocksdb_env_t* env; rocksdb_env_t* env;
rocksdb_options_t* options; rocksdb_options_t* options;
rocksdb_compactoptions_t* coptions; rocksdb_compactoptions_t* coptions;
@ -649,30 +635,20 @@ int main(int argc, char** argv) {
char* err = NULL; char* err = NULL;
int run = -1; int run = -1;
snprintf(dbname, sizeof(dbname), snprintf(dbname, sizeof(dbname), "%s/rocksdb_c_test-%d", GetTempDir(),
"%s/rocksdb_c_test-%d", ((int)geteuid()));
GetTempDir(),
((int) geteuid()));
snprintf(dbbackupname, sizeof(dbbackupname), snprintf(dbbackupname, sizeof(dbbackupname), "%s/rocksdb_c_test-%d-backup",
"%s/rocksdb_c_test-%d-backup", GetTempDir(), ((int)geteuid()));
GetTempDir(),
((int) geteuid()));
snprintf(dbcheckpointname, sizeof(dbcheckpointname), snprintf(dbcheckpointname, sizeof(dbcheckpointname),
"%s/rocksdb_c_test-%d-checkpoint", "%s/rocksdb_c_test-%d-checkpoint", GetTempDir(), ((int)geteuid()));
GetTempDir(),
((int) geteuid()));
snprintf(sstfilename, sizeof(sstfilename), snprintf(sstfilename, sizeof(sstfilename), "%s/rocksdb_c_test-%d-sst",
"%s/rocksdb_c_test-%d-sst", GetTempDir(), ((int)geteuid()));
GetTempDir(),
((int)geteuid()));
snprintf(dbpathname, sizeof(dbpathname), snprintf(dbpathname, sizeof(dbpathname), "%s/rocksdb_c_test-%d-dbpath",
"%s/rocksdb_c_test-%d-dbpath", GetTempDir(), ((int)geteuid()));
GetTempDir(),
((int) geteuid()));
StartPhase("create_objects"); StartPhase("create_objects");
cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
@ -746,7 +722,8 @@ int main(int argc, char** argv) {
rocksdb_destroy_db(options, dbbackupname, &err); rocksdb_destroy_db(options, dbbackupname, &err);
CheckNoError(err); CheckNoError(err);
rocksdb_backup_engine_t *be = rocksdb_backup_engine_open(options, dbbackupname, &err); rocksdb_backup_engine_t* be =
rocksdb_backup_engine_open(options, dbbackupname, &err);
CheckNoError(err); CheckNoError(err);
rocksdb_backup_engine_create_new_backup(be, db, &err); rocksdb_backup_engine_create_new_backup(be, db, &err);
@ -759,7 +736,8 @@ int main(int argc, char** argv) {
rocksdb_backup_engine_create_new_backup(be, db, &err); rocksdb_backup_engine_create_new_backup(be, db, &err);
CheckNoError(err); CheckNoError(err);
const rocksdb_backup_engine_info_t* bei = rocksdb_backup_engine_get_backup_info(be); const rocksdb_backup_engine_info_t* bei =
rocksdb_backup_engine_get_backup_info(be);
CheckCondition(rocksdb_backup_engine_info_count(bei) > 1); CheckCondition(rocksdb_backup_engine_info_count(bei) > 1);
rocksdb_backup_engine_info_destroy(bei); rocksdb_backup_engine_info_destroy(bei);
@ -778,9 +756,11 @@ int main(int argc, char** argv) {
rocksdb_destroy_db(options, dbname, &err); rocksdb_destroy_db(options, dbname, &err);
CheckNoError(err); CheckNoError(err);
rocksdb_restore_options_t *restore_options = rocksdb_restore_options_create(); rocksdb_restore_options_t* restore_options =
rocksdb_restore_options_create();
rocksdb_restore_options_set_keep_log_files(restore_options, 0); rocksdb_restore_options_set_keep_log_files(restore_options, 0);
rocksdb_backup_engine_restore_db_from_latest_backup(be, dbname, dbname, restore_options, &err); rocksdb_backup_engine_restore_db_from_latest_backup(be, dbname, dbname,
restore_options, &err);
CheckNoError(err); CheckNoError(err);
rocksdb_restore_options_destroy(restore_options); rocksdb_restore_options_destroy(restore_options);
@ -799,7 +779,8 @@ int main(int argc, char** argv) {
rocksdb_destroy_db(options, dbcheckpointname, &err); rocksdb_destroy_db(options, dbcheckpointname, &err);
CheckNoError(err); CheckNoError(err);
rocksdb_checkpoint_t* checkpoint = rocksdb_checkpoint_object_create(db, &err); rocksdb_checkpoint_t* checkpoint =
rocksdb_checkpoint_object_create(db, &err);
CheckNoError(err); CheckNoError(err);
rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err); rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err);
@ -976,10 +957,10 @@ int main(int argc, char** argv) {
StartPhase("writebatch_vectors"); StartPhase("writebatch_vectors");
{ {
rocksdb_writebatch_t* wb = rocksdb_writebatch_create(); rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
const char* k_list[2] = { "z", "ap" }; const char* k_list[2] = {"z", "ap"};
const size_t k_sizes[2] = { 1, 2 }; const size_t k_sizes[2] = {1, 2};
const char* v_list[3] = { "x", "y", "z" }; const char* v_list[3] = {"x", "y", "z"};
const size_t v_sizes[3] = { 1, 1, 1 }; const size_t v_sizes[3] = {1, 1, 1};
rocksdb_writebatch_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes); rocksdb_writebatch_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
rocksdb_write(db, woptions, wb, &err); rocksdb_write(db, woptions, wb, &err);
CheckNoError(err); CheckNoError(err);
@ -1041,13 +1022,17 @@ int main(int argc, char** argv) {
CheckCondition(count == 3); CheckCondition(count == 3);
size_t size; size_t size;
char* value; char* value;
value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "box", 3, &size, &err); value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "box", 3, &size,
&err);
CheckValue(err, "c", &value, size); CheckValue(err, "c", &value, size);
value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "bar", 3, &size, &err); value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "bar", 3, &size,
&err);
CheckValue(err, NULL, &value, size); CheckValue(err, NULL, &value, size);
value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, "foo", 3, &size, &err); value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions,
"foo", 3, &size, &err);
CheckValue(err, "hello", &value, size); CheckValue(err, "hello", &value, size);
value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, "box", 3, &size, &err); value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions,
"box", 3, &size, &err);
CheckValue(err, "c", &value, size); CheckValue(err, "c", &value, size);
rocksdb_write_writebatch_wi(db, woptions, wbi, &err); rocksdb_write_writebatch_wi(db, woptions, wbi, &err);
CheckNoError(err); CheckNoError(err);
@ -1064,10 +1049,10 @@ int main(int argc, char** argv) {
StartPhase("writebatch_wi_vectors"); StartPhase("writebatch_wi_vectors");
{ {
rocksdb_writebatch_wi_t* wb = rocksdb_writebatch_wi_create(0, 1); rocksdb_writebatch_wi_t* wb = rocksdb_writebatch_wi_create(0, 1);
const char* k_list[2] = { "z", "ap" }; const char* k_list[2] = {"z", "ap"};
const size_t k_sizes[2] = { 1, 2 }; const size_t k_sizes[2] = {1, 2};
const char* v_list[3] = { "x", "y", "z" }; const char* v_list[3] = {"x", "y", "z"};
const size_t v_sizes[3] = { 1, 1, 1 }; const size_t v_sizes[3] = {1, 1, 1};
rocksdb_writebatch_wi_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes); rocksdb_writebatch_wi_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
rocksdb_write_writebatch_wi(db, woptions, wb, &err); rocksdb_write_writebatch_wi(db, woptions, wb, &err);
CheckNoError(err); CheckNoError(err);
@ -1156,13 +1141,14 @@ int main(int argc, char** argv) {
StartPhase("multiget"); StartPhase("multiget");
{ {
const char* keys[3] = { "box", "foo", "notfound" }; const char* keys[3] = {"box", "foo", "notfound"};
const size_t keys_sizes[3] = { 3, 3, 8 }; const size_t keys_sizes[3] = {3, 3, 8};
char* vals[3]; char* vals[3];
size_t vals_sizes[3]; size_t vals_sizes[3];
char* errs[3]; char* errs[3];
const char* expected[3] = {"c", "hello", NULL}; const char* expected[3] = {"c", "hello", NULL};
rocksdb_multi_get(db, roptions, 3, keys, keys_sizes, vals, vals_sizes, errs); rocksdb_multi_get(db, roptions, 3, keys, keys_sizes, vals, vals_sizes,
errs);
CheckMultiGetValues(3, vals, vals_sizes, errs, expected); CheckMultiGetValues(3, vals, vals_sizes, errs, expected);
} }
@ -1180,10 +1166,10 @@ int main(int argc, char** argv) {
char keybuf[100]; char keybuf[100];
char valbuf[100]; char valbuf[100];
uint64_t sizes[2]; uint64_t sizes[2];
const char* start[2] = { "a", "k00000000000000010000" }; const char* start[2] = {"a", "k00000000000000010000"};
size_t start_len[2] = { 1, 21 }; size_t start_len[2] = {1, 21};
const char* limit[2] = { "k00000000000000010000", "z" }; const char* limit[2] = {"k00000000000000010000", "z"};
size_t limit_len[2] = { 21, 1 }; size_t limit_len[2] = {21, 1};
rocksdb_writeoptions_set_sync(woptions, 0); rocksdb_writeoptions_set_sync(woptions, 0);
for (i = 0; i < n; i++) { for (i = 0; i < n; i++) {
snprintf(keybuf, sizeof(keybuf), "k%020d", i); snprintf(keybuf, sizeof(keybuf), "k%020d", i);
@ -1393,8 +1379,8 @@ int main(int argc, char** argv) {
factory); factory);
db = CheckCompaction(db, options_with_filter_factory, roptions, woptions); db = CheckCompaction(db, options_with_filter_factory, roptions, woptions);
rocksdb_options_set_compaction_filter_factory( rocksdb_options_set_compaction_filter_factory(options_with_filter_factory,
options_with_filter_factory, NULL); NULL);
rocksdb_options_destroy(options_with_filter_factory); rocksdb_options_destroy(options_with_filter_factory);
} }
@ -1449,7 +1435,8 @@ int main(int argc, char** argv) {
rocksdb_close(db); rocksdb_close(db);
size_t cflen; size_t cflen;
char** column_fams = rocksdb_list_column_families(db_options, dbname, &cflen, &err); char** column_fams =
rocksdb_list_column_families(db_options, dbname, &cflen, &err);
CheckNoError(err); CheckNoError(err);
CheckEqual("default", column_fams[0], 7); CheckEqual("default", column_fams[0], 7);
CheckEqual("cf1", column_fams[1], 3); CheckEqual("cf1", column_fams[1], 3);
@ -1465,7 +1452,8 @@ int main(int argc, char** argv) {
LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 2, cf_names, LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 2, cf_names,
NULL); NULL);
db = rocksdb_open_column_families(db_options, dbname, 2, cf_names, cf_opts, handles, &err); db = rocksdb_open_column_families(db_options, dbname, 2, cf_names, cf_opts,
handles, &err);
CheckNoError(err); CheckNoError(err);
rocksdb_put_cf(db, woptions, handles[1], "foo", 3, "hello", 5, &err); rocksdb_put_cf(db, woptions, handles[1], "foo", 3, "hello", 5, &err);
@ -1483,11 +1471,10 @@ int main(int argc, char** argv) {
&err); &err);
CheckNoError(err); CheckNoError(err);
rocksdb_flushoptions_t *flush_options = rocksdb_flushoptions_create(); rocksdb_flushoptions_t* flush_options = rocksdb_flushoptions_create();
rocksdb_flushoptions_set_wait(flush_options, 1); rocksdb_flushoptions_set_wait(flush_options, 1);
rocksdb_flush_cf(db, flush_options, handles[1], &err); rocksdb_flush_cf(db, flush_options, handles[1], &err);
CheckNoError(err) CheckNoError(err) rocksdb_flushoptions_destroy(flush_options);
rocksdb_flushoptions_destroy(flush_options);
CheckGetCF(db, roptions, handles[1], "foo", "hello"); CheckGetCF(db, roptions, handles[1], "foo", "hello");
CheckPinGetCF(db, roptions, handles[1], "foo", "hello"); CheckPinGetCF(db, roptions, handles[1], "foo", "hello");
@ -1524,27 +1511,29 @@ int main(int argc, char** argv) {
rocksdb_flush_wal(db, 1, &err); rocksdb_flush_wal(db, 1, &err);
CheckNoError(err); CheckNoError(err);
const char* keys[3] = { "box", "box", "barfooxx" }; const char* keys[3] = {"box", "box", "barfooxx"};
const rocksdb_column_family_handle_t* get_handles[3] = { handles[0], handles[1], handles[1] }; const rocksdb_column_family_handle_t* get_handles[3] = {
const size_t keys_sizes[3] = { 3, 3, 8 }; handles[0], handles[1], handles[1]};
const size_t keys_sizes[3] = {3, 3, 8};
char* vals[3]; char* vals[3];
size_t vals_sizes[3]; size_t vals_sizes[3];
char* errs[3]; char* errs[3];
rocksdb_multi_get_cf(db, roptions, get_handles, 3, keys, keys_sizes, vals, vals_sizes, errs); rocksdb_multi_get_cf(db, roptions, get_handles, 3, keys, keys_sizes, vals,
vals_sizes, errs);
int i; int i;
for (i = 0; i < 3; i++) { for (i = 0; i < 3; i++) {
CheckEqual(NULL, errs[i], 0); CheckEqual(NULL, errs[i], 0);
switch (i) { switch (i) {
case 0: case 0:
CheckEqual(NULL, vals[i], vals_sizes[i]); // wrong cf CheckEqual(NULL, vals[i], vals_sizes[i]); // wrong cf
break; break;
case 1: case 1:
CheckEqual("c", vals[i], vals_sizes[i]); // bingo CheckEqual("c", vals[i], vals_sizes[i]); // bingo
break; break;
case 2: case 2:
CheckEqual(NULL, vals[i], vals_sizes[i]); // normal not found CheckEqual(NULL, vals[i], vals_sizes[i]); // normal not found
break; break;
} }
Free(&vals[i]); Free(&vals[i]);
} }
@ -1592,7 +1581,8 @@ int main(int argc, char** argv) {
} }
} }
rocksdb_iterator_t* iter = rocksdb_create_iterator_cf(db, roptions, handles[1]); rocksdb_iterator_t* iter =
rocksdb_create_iterator_cf(db, roptions, handles[1]);
CheckCondition(!rocksdb_iter_valid(iter)); CheckCondition(!rocksdb_iter_valid(iter));
rocksdb_iter_seek_to_first(iter); rocksdb_iter_seek_to_first(iter);
CheckCondition(rocksdb_iter_valid(iter)); CheckCondition(rocksdb_iter_valid(iter));
@ -1605,9 +1595,11 @@ int main(int argc, char** argv) {
CheckNoError(err); CheckNoError(err);
rocksdb_iter_destroy(iter); rocksdb_iter_destroy(iter);
rocksdb_column_family_handle_t* iters_cf_handles[2] = { handles[0], handles[1] }; rocksdb_column_family_handle_t* iters_cf_handles[2] = {handles[0],
handles[1]};
rocksdb_iterator_t* iters_handles[2]; rocksdb_iterator_t* iters_handles[2];
rocksdb_create_iterators(db, roptions, iters_cf_handles, iters_handles, 2, &err); rocksdb_create_iterators(db, roptions, iters_cf_handles, iters_handles, 2,
&err);
CheckNoError(err); CheckNoError(err);
iter = iters_handles[0]; iter = iters_handles[0];
@ -1652,7 +1644,8 @@ int main(int argc, char** argv) {
{ {
// Create new database // Create new database
rocksdb_options_set_allow_mmap_reads(options, 1); rocksdb_options_set_allow_mmap_reads(options, 1);
rocksdb_options_set_prefix_extractor(options, rocksdb_slicetransform_create_fixed_prefix(3)); rocksdb_options_set_prefix_extractor(
options, rocksdb_slicetransform_create_fixed_prefix(3));
rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4); rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4);
rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16); rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16);
rocksdb_options_set_allow_concurrent_memtable_write(options, 0); rocksdb_options_set_allow_concurrent_memtable_write(options, 0);
@ -1747,8 +1740,9 @@ int main(int argc, char** argv) {
// amount of memory used within memtables should grow // amount of memory used within memtables should grow
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_total(mu2) >= CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_total(mu2) >=
rocksdb_approximate_memory_usage_get_mem_table_total(mu1)); rocksdb_approximate_memory_usage_get_mem_table_total(mu1));
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu2) >= CheckCondition(
rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu1)); rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu2) >=
rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu1));
rocksdb_memory_consumers_destroy(consumers); rocksdb_memory_consumers_destroy(consumers);
rocksdb_approximate_memory_usage_destroy(mu1); rocksdb_approximate_memory_usage_destroy(mu1);
@ -2839,53 +2833,57 @@ int main(int argc, char** argv) {
db = rocksdb_open(options, dbname, &err); db = rocksdb_open(options, dbname, &err);
CheckNoError(err); CheckNoError(err);
rocksdb_put(db, woptions, "a", 1, "0", 1, &err); CheckNoError(err); rocksdb_put(db, woptions, "a", 1, "0", 1, &err);
rocksdb_put(db, woptions, "foo", 3, "bar", 3, &err); CheckNoError(err); CheckNoError(err);
rocksdb_put(db, woptions, "foo1", 4, "bar1", 4, &err); CheckNoError(err); rocksdb_put(db, woptions, "foo", 3, "bar", 3, &err);
rocksdb_put(db, woptions, "g1", 2, "0", 1, &err); CheckNoError(err); CheckNoError(err);
rocksdb_put(db, woptions, "foo1", 4, "bar1", 4, &err);
CheckNoError(err);
rocksdb_put(db, woptions, "g1", 2, "0", 1, &err);
CheckNoError(err);
// testing basic case with no iterate_upper_bound and no prefix_extractor // testing basic case with no iterate_upper_bound and no prefix_extractor
{ {
rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0); rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0);
rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions); rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
rocksdb_iter_seek(iter, "foo", 3); rocksdb_iter_seek(iter, "foo", 3);
CheckCondition(rocksdb_iter_valid(iter)); CheckCondition(rocksdb_iter_valid(iter));
CheckIter(iter, "foo", "bar"); CheckIter(iter, "foo", "bar");
rocksdb_iter_next(iter); rocksdb_iter_next(iter);
CheckCondition(rocksdb_iter_valid(iter)); CheckCondition(rocksdb_iter_valid(iter));
CheckIter(iter, "foo1", "bar1"); CheckIter(iter, "foo1", "bar1");
rocksdb_iter_next(iter); rocksdb_iter_next(iter);
CheckCondition(rocksdb_iter_valid(iter)); CheckCondition(rocksdb_iter_valid(iter));
CheckIter(iter, "g1", "0"); CheckIter(iter, "g1", "0");
rocksdb_iter_destroy(iter); rocksdb_iter_destroy(iter);
} }
// testing iterate_upper_bound and forward iterator // testing iterate_upper_bound and forward iterator
// to make sure it stops at bound // to make sure it stops at bound
{ {
// iterate_upper_bound points beyond the last expected entry // iterate_upper_bound points beyond the last expected entry
rocksdb_readoptions_set_iterate_upper_bound(roptions, "foo2", 4); rocksdb_readoptions_set_iterate_upper_bound(roptions, "foo2", 4);
rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions); rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
rocksdb_iter_seek(iter, "foo", 3); rocksdb_iter_seek(iter, "foo", 3);
CheckCondition(rocksdb_iter_valid(iter)); CheckCondition(rocksdb_iter_valid(iter));
CheckIter(iter, "foo", "bar"); CheckIter(iter, "foo", "bar");
rocksdb_iter_next(iter); rocksdb_iter_next(iter);
CheckCondition(rocksdb_iter_valid(iter)); CheckCondition(rocksdb_iter_valid(iter));
CheckIter(iter, "foo1", "bar1"); CheckIter(iter, "foo1", "bar1");
rocksdb_iter_next(iter); rocksdb_iter_next(iter);
// should stop here... // should stop here...
CheckCondition(!rocksdb_iter_valid(iter)); CheckCondition(!rocksdb_iter_valid(iter));
rocksdb_iter_destroy(iter); rocksdb_iter_destroy(iter);
rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0); rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0);
} }
} }
@ -3009,7 +3007,7 @@ int main(int argc, char** argv) {
snapshot = rocksdb_transactiondb_create_snapshot(txn_db); snapshot = rocksdb_transactiondb_create_snapshot(txn_db);
rocksdb_readoptions_set_snapshot(roptions, snapshot); rocksdb_readoptions_set_snapshot(roptions, snapshot);
rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hey", 3, &err); rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hey", 3, &err);
CheckNoError(err); CheckNoError(err);
CheckTxnDBGet(txn_db, roptions, "foo", "hello"); CheckTxnDBGet(txn_db, roptions, "foo", "hello");
@ -3021,7 +3019,8 @@ int main(int argc, char** argv) {
// iterate // iterate
rocksdb_transaction_put(txn, "bar", 3, "hi", 2, &err); rocksdb_transaction_put(txn, "bar", 3, "hi", 2, &err);
rocksdb_iterator_t* iter = rocksdb_transaction_create_iterator(txn, roptions); rocksdb_iterator_t* iter =
rocksdb_transaction_create_iterator(txn, roptions);
CheckCondition(!rocksdb_iter_valid(iter)); CheckCondition(!rocksdb_iter_valid(iter));
rocksdb_iter_seek_to_first(iter); rocksdb_iter_seek_to_first(iter);
CheckCondition(rocksdb_iter_valid(iter)); CheckCondition(rocksdb_iter_valid(iter));

@ -192,8 +192,7 @@ Status CheckCFPathsSupported(const DBOptions& db_options,
return Status::NotSupported( return Status::NotSupported(
"More than one CF paths are only supported in " "More than one CF paths are only supported in "
"universal and level compaction styles. "); "universal and level compaction styles. ");
} else if (cf_options.cf_paths.empty() && } else if (cf_options.cf_paths.empty() && db_options.db_paths.size() > 1) {
db_options.db_paths.size() > 1) {
return Status::NotSupported( return Status::NotSupported(
"More than one DB paths are only supported in " "More than one DB paths are only supported in "
"universal and level compaction styles. "); "universal and level compaction styles. ");
@ -205,7 +204,7 @@ Status CheckCFPathsSupported(const DBOptions& db_options,
namespace { namespace {
const uint64_t kDefaultTtl = 0xfffffffffffffffe; const uint64_t kDefaultTtl = 0xfffffffffffffffe;
const uint64_t kDefaultPeriodicCompSecs = 0xfffffffffffffffe; const uint64_t kDefaultPeriodicCompSecs = 0xfffffffffffffffe;
} // namespace } // anonymous namespace
ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options, ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
const ColumnFamilyOptions& src) { const ColumnFamilyOptions& src) {
@ -353,7 +352,8 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
// were not deleted yet, when we open the DB we will find these .trash files // were not deleted yet, when we open the DB we will find these .trash files
// and schedule them to be deleted (or delete immediately if SstFileManager // and schedule them to be deleted (or delete immediately if SstFileManager
// was not used) // was not used)
auto sfm = static_cast<SstFileManagerImpl*>(db_options.sst_file_manager.get()); auto sfm =
static_cast<SstFileManagerImpl*>(db_options.sst_file_manager.get());
for (size_t i = 0; i < result.cf_paths.size(); i++) { for (size_t i = 0; i < result.cf_paths.size(); i++) {
DeleteScheduler::CleanupDirectory(db_options.env, sfm, DeleteScheduler::CleanupDirectory(db_options.env, sfm,
result.cf_paths[i].path) result.cf_paths[i].path)
@ -610,8 +610,8 @@ ColumnFamilyData::ColumnFamilyData(
compaction_picker_.reset( compaction_picker_.reset(
new FIFOCompactionPicker(ioptions_, &internal_comparator_)); new FIFOCompactionPicker(ioptions_, &internal_comparator_));
} else if (ioptions_.compaction_style == kCompactionStyleNone) { } else if (ioptions_.compaction_style == kCompactionStyleNone) {
compaction_picker_.reset(new NullCompactionPicker( compaction_picker_.reset(
ioptions_, &internal_comparator_)); new NullCompactionPicker(ioptions_, &internal_comparator_));
ROCKS_LOG_WARN(ioptions_.logger, ROCKS_LOG_WARN(ioptions_.logger,
"Column family %s does not use any background compaction. " "Column family %s does not use any background compaction. "
"Compactions can only be done via CompactFiles\n", "Compactions can only be done via CompactFiles\n",
@ -878,7 +878,7 @@ int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
return static_cast<int>(res); return static_cast<int>(res);
} }
} }
} // namespace } // anonymous namespace
std::pair<WriteStallCondition, ColumnFamilyData::WriteStallCause> std::pair<WriteStallCondition, ColumnFamilyData::WriteStallCause>
ColumnFamilyData::GetWriteStallConditionAndCause( ColumnFamilyData::GetWriteStallConditionAndCause(
@ -919,7 +919,7 @@ ColumnFamilyData::GetWriteStallConditionAndCause(
} }
WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions( WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
const MutableCFOptions& mutable_cf_options) { const MutableCFOptions& mutable_cf_options) {
auto write_stall_condition = WriteStallCondition::kNormal; auto write_stall_condition = WriteStallCondition::kNormal;
if (current_ != nullptr) { if (current_ != nullptr) {
auto* vstorage = current_->storage_info(); auto* vstorage = current_->storage_info();
@ -1012,7 +1012,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
mutable_cf_options.hard_pending_compaction_bytes_limit > 0 && mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
(compaction_needed_bytes - (compaction_needed_bytes -
mutable_cf_options.soft_pending_compaction_bytes_limit) > mutable_cf_options.soft_pending_compaction_bytes_limit) >
3 * (mutable_cf_options.hard_pending_compaction_bytes_limit - 3 *
(mutable_cf_options.hard_pending_compaction_bytes_limit -
mutable_cf_options.soft_pending_compaction_bytes_limit) / mutable_cf_options.soft_pending_compaction_bytes_limit) /
4; 4;
@ -1305,8 +1306,8 @@ bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) {
return false; return false;
} }
void ColumnFamilyData::InstallSuperVersion( void ColumnFamilyData::InstallSuperVersion(SuperVersionContext* sv_context,
SuperVersionContext* sv_context, InstrumentedMutex* db_mutex) { InstrumentedMutex* db_mutex) {
db_mutex->AssertHeld(); db_mutex->AssertHeld();
return InstallSuperVersion(sv_context, mutable_cf_options_); return InstallSuperVersion(sv_context, mutable_cf_options_);
} }
@ -1483,8 +1484,8 @@ Env::WriteLifeTimeHint ColumnFamilyData::CalculateSSTWriteHint(int level) {
// than base_level. // than base_level.
return Env::WLTH_MEDIUM; return Env::WLTH_MEDIUM;
} }
return static_cast<Env::WriteLifeTimeHint>(level - base_level + return static_cast<Env::WriteLifeTimeHint>(
static_cast<int>(Env::WLTH_MEDIUM)); level - base_level + static_cast<int>(Env::WLTH_MEDIUM));
} }
Status ColumnFamilyData::AddDirectories( Status ColumnFamilyData::AddDirectories(
@ -1580,8 +1581,8 @@ ColumnFamilyData* ColumnFamilySet::GetColumnFamily(uint32_t id) const {
} }
} }
ColumnFamilyData* ColumnFamilySet::GetColumnFamily(const std::string& name) ColumnFamilyData* ColumnFamilySet::GetColumnFamily(
const { const std::string& name) const {
auto cfd_iter = column_families_.find(name); auto cfd_iter = column_families_.find(name);
if (cfd_iter != column_families_.end()) { if (cfd_iter != column_families_.end()) {
auto cfd = GetColumnFamily(cfd_iter->second); auto cfd = GetColumnFamily(cfd_iter->second);

@ -163,8 +163,8 @@ extern const double kIncSlowdownRatio;
class ColumnFamilyHandleImpl : public ColumnFamilyHandle { class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
public: public:
// create while holding the mutex // create while holding the mutex
ColumnFamilyHandleImpl( ColumnFamilyHandleImpl(ColumnFamilyData* cfd, DBImpl* db,
ColumnFamilyData* cfd, DBImpl* db, InstrumentedMutex* mutex); InstrumentedMutex* mutex);
// destroy without mutex // destroy without mutex
virtual ~ColumnFamilyHandleImpl(); virtual ~ColumnFamilyHandleImpl();
virtual ColumnFamilyData* cfd() const { return cfd_; } virtual ColumnFamilyData* cfd() const { return cfd_; }
@ -189,7 +189,8 @@ class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl { class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
public: public:
ColumnFamilyHandleInternal() ColumnFamilyHandleInternal()
: ColumnFamilyHandleImpl(nullptr, nullptr, nullptr), internal_cfd_(nullptr) {} : ColumnFamilyHandleImpl(nullptr, nullptr, nullptr),
internal_cfd_(nullptr) {}
void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; } void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; }
virtual ColumnFamilyData* cfd() const override { return internal_cfd_; } virtual ColumnFamilyData* cfd() const override { return internal_cfd_; }
@ -357,7 +358,7 @@ class ColumnFamilyData {
Version* current() { return current_; } Version* current() { return current_; }
Version* dummy_versions() { return dummy_versions_; } Version* dummy_versions() { return dummy_versions_; }
void SetCurrent(Version* _current); void SetCurrent(Version* _current);
uint64_t GetNumLiveVersions() const; // REQUIRE: DB mutex held uint64_t GetNumLiveVersions() const; // REQUIRE: DB mutex held
uint64_t GetTotalSstFilesSize() const; // REQUIRE: DB mutex held uint64_t GetTotalSstFilesSize() const; // REQUIRE: DB mutex held
uint64_t GetLiveSstFilesSize() const; // REQUIRE: DB mutex held uint64_t GetLiveSstFilesSize() const; // REQUIRE: DB mutex held
uint64_t GetTotalBlobFileSize() const; // REQUIRE: DB mutex held uint64_t GetTotalBlobFileSize() const; // REQUIRE: DB mutex held
@ -552,7 +553,7 @@ class ColumnFamilyData {
Version* dummy_versions_; // Head of circular doubly-linked list of versions. Version* dummy_versions_; // Head of circular doubly-linked list of versions.
Version* current_; // == dummy_versions->prev_ Version* current_; // == dummy_versions->prev_
std::atomic<int> refs_; // outstanding references to ColumnFamilyData std::atomic<int> refs_; // outstanding references to ColumnFamilyData
std::atomic<bool> initialized_; std::atomic<bool> initialized_;
std::atomic<bool> dropped_; // true if client dropped it std::atomic<bool> dropped_; // true if client dropped it
@ -656,8 +657,7 @@ class ColumnFamilySet {
// ColumnFamilySet supports iteration // ColumnFamilySet supports iteration
class iterator { class iterator {
public: public:
explicit iterator(ColumnFamilyData* cfd) explicit iterator(ColumnFamilyData* cfd) : current_(cfd) {}
: current_(cfd) {}
// NOTE: minimum operators for for-loop iteration // NOTE: minimum operators for for-loop iteration
iterator& operator++() { iterator& operator++() {
current_ = current_->next_; current_ = current_->next_;

@ -39,9 +39,7 @@ class EnvCounter : public SpecialEnv {
public: public:
explicit EnvCounter(Env* base) explicit EnvCounter(Env* base)
: SpecialEnv(base), num_new_writable_file_(0) {} : SpecialEnv(base), num_new_writable_file_(0) {}
int GetNumberOfNewWritableFileCalls() { int GetNumberOfNewWritableFileCalls() { return num_new_writable_file_; }
return num_new_writable_file_;
}
Status NewWritableFile(const std::string& f, std::unique_ptr<WritableFile>* r, Status NewWritableFile(const std::string& f, std::unique_ptr<WritableFile>* r,
const EnvOptions& soptions) override { const EnvOptions& soptions) override {
++num_new_writable_file_; ++num_new_writable_file_;
@ -187,7 +185,7 @@ class ColumnFamilyTestBase : public testing::Test {
} }
Status OpenReadOnly(std::vector<std::string> cf, Status OpenReadOnly(std::vector<std::string> cf,
std::vector<ColumnFamilyOptions> options = {}) { std::vector<ColumnFamilyOptions> options = {}) {
std::vector<ColumnFamilyDescriptor> column_families; std::vector<ColumnFamilyDescriptor> column_families;
names_.clear(); names_.clear();
for (size_t i = 0; i < cf.size(); ++i) { for (size_t i = 0; i < cf.size(); ++i) {
@ -201,20 +199,17 @@ class ColumnFamilyTestBase : public testing::Test {
#ifndef ROCKSDB_LITE // ReadOnlyDB is not supported #ifndef ROCKSDB_LITE // ReadOnlyDB is not supported
void AssertOpenReadOnly(std::vector<std::string> cf, void AssertOpenReadOnly(std::vector<std::string> cf,
std::vector<ColumnFamilyOptions> options = {}) { std::vector<ColumnFamilyOptions> options = {}) {
ASSERT_OK(OpenReadOnly(cf, options)); ASSERT_OK(OpenReadOnly(cf, options));
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
void Open(std::vector<std::string> cf, void Open(std::vector<std::string> cf,
std::vector<ColumnFamilyOptions> options = {}) { std::vector<ColumnFamilyOptions> options = {}) {
ASSERT_OK(TryOpen(cf, options)); ASSERT_OK(TryOpen(cf, options));
} }
void Open() { void Open() { Open({"default"}); }
Open({"default"});
}
DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); } DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
@ -253,7 +248,7 @@ class ColumnFamilyTestBase : public testing::Test {
} }
void Destroy(const std::vector<ColumnFamilyDescriptor>& column_families = void Destroy(const std::vector<ColumnFamilyDescriptor>& column_families =
std::vector<ColumnFamilyDescriptor>()) { std::vector<ColumnFamilyDescriptor>()) {
Close(); Close();
ASSERT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_), ASSERT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_),
column_families)); column_families));
@ -335,9 +330,7 @@ class ColumnFamilyTestBase : public testing::Test {
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf])); ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf]));
} }
void WaitForCompaction() { void WaitForCompaction() { ASSERT_OK(dbfull()->TEST_WaitForCompact()); }
ASSERT_OK(dbfull()->TEST_WaitForCompact());
}
uint64_t MaxTotalInMemoryState() { uint64_t MaxTotalInMemoryState() {
return dbfull()->TEST_MaxTotalInMemoryState(); return dbfull()->TEST_MaxTotalInMemoryState();
@ -354,9 +347,7 @@ class ColumnFamilyTestBase : public testing::Test {
Status Merge(int cf, const std::string& key, const std::string& value) { Status Merge(int cf, const std::string& key, const std::string& value) {
return db_->Merge(WriteOptions(), handles_[cf], Slice(key), Slice(value)); return db_->Merge(WriteOptions(), handles_[cf], Slice(key), Slice(value));
} }
Status Flush(int cf) { Status Flush(int cf) { return db_->Flush(FlushOptions(), handles_[cf]); }
return db_->Flush(FlushOptions(), handles_[cf]);
}
std::string Get(int cf, const std::string& key) { std::string Get(int cf, const std::string& key) {
ReadOptions options; ReadOptions options;
@ -409,8 +400,8 @@ class ColumnFamilyTestBase : public testing::Test {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
ASSERT_EQ(value, FilesPerLevel(cf)); ASSERT_EQ(value, FilesPerLevel(cf));
#else #else
(void) value; (void)value;
(void) cf; (void)cf;
#endif #endif
} }
@ -426,7 +417,7 @@ class ColumnFamilyTestBase : public testing::Test {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
ASSERT_EQ(expected_value, CountLiveFiles()); ASSERT_EQ(expected_value, CountLiveFiles());
#else #else
(void) expected_value; (void)expected_value;
#endif #endif
} }
@ -476,7 +467,7 @@ class ColumnFamilyTestBase : public testing::Test {
#ifndef ROCKSDB_LITE // GetSortedWalFiles is not supported #ifndef ROCKSDB_LITE // GetSortedWalFiles is not supported
ASSERT_EQ(value, CountLiveLogFiles()); ASSERT_EQ(value, CountLiveLogFiles());
#else #else
(void) value; (void)value;
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
} }
@ -521,14 +512,14 @@ class ColumnFamilyTestBase : public testing::Test {
return static_cast<int>(files.size()); return static_cast<int>(files.size());
} }
void RecalculateWriteStallConditions(ColumnFamilyData* cfd, void RecalculateWriteStallConditions(
const MutableCFOptions& mutable_cf_options) { ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options) {
// add lock to avoid race condition between // add lock to avoid race condition between
// `RecalculateWriteStallConditions` which writes to CFStats and // `RecalculateWriteStallConditions` which writes to CFStats and
// background `DBImpl::DumpStats()` threads which read CFStats // background `DBImpl::DumpStats()` threads which read CFStats
dbfull()->TEST_LockMutex(); dbfull()->TEST_LockMutex();
cfd->RecalculateWriteStallConditions(mutable_cf_options); cfd->RecalculateWriteStallConditions(mutable_cf_options);
dbfull()-> TEST_UnlockMutex(); dbfull()->TEST_UnlockMutex();
} }
std::vector<ColumnFamilyHandle*> handles_; std::vector<ColumnFamilyHandle*> handles_;
@ -970,8 +961,7 @@ TEST_P(ColumnFamilyTest, FlushTest) {
} }
for (int i = 0; i < 3; ++i) { for (int i = 0; i < 3; ++i) {
uint64_t max_total_in_memory_state = uint64_t max_total_in_memory_state = MaxTotalInMemoryState();
MaxTotalInMemoryState();
ASSERT_OK(Flush(i)); ASSERT_OK(Flush(i));
AssertMaxTotalInMemoryState(max_total_in_memory_state); AssertMaxTotalInMemoryState(max_total_in_memory_state);
} }
@ -1209,7 +1199,7 @@ TEST_P(ColumnFamilyTest, DifferentWriteBufferSizes) {
WaitForFlush(2); WaitForFlush(2);
AssertNumberOfImmutableMemtables({0, 0, 0, 0}); AssertNumberOfImmutableMemtables({0, 0, 0, 0});
AssertCountLiveLogFiles(12); AssertCountLiveLogFiles(12);
PutRandomData(1, 2*200, 1000); PutRandomData(1, 2 * 200, 1000);
WaitForFlush(1); WaitForFlush(1);
AssertNumberOfImmutableMemtables({0, 0, 0, 0}); AssertNumberOfImmutableMemtables({0, 0, 0, 0});
AssertCountLiveLogFiles(7); AssertCountLiveLogFiles(7);
@ -2123,7 +2113,6 @@ TEST_P(ColumnFamilyTest, ReadOnlyDBTest) {
ASSERT_EQ("bla", Get(1, "foo")); ASSERT_EQ("bla", Get(1, "foo"));
ASSERT_EQ("blablablabla", Get(2, "foo")); ASSERT_EQ("blablablabla", Get(2, "foo"));
// test newiterators // test newiterators
{ {
std::vector<Iterator*> iterators; std::vector<Iterator*> iterators;
@ -2488,7 +2477,7 @@ void DropSingleColumnFamily(ColumnFamilyTest* cf_test, int cf_id,
} }
test_stage = kChildThreadFinishDroppingColumnFamily; test_stage = kChildThreadFinishDroppingColumnFamily;
} }
} // namespace } // anonymous namespace
TEST_P(ColumnFamilyTest, CreateAndDropRace) { TEST_P(ColumnFamilyTest, CreateAndDropRace) {
const int kCfCount = 5; const int kCfCount = 5;

@ -348,9 +348,7 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
return true; return true;
} }
void SetDB(DB* db) { void SetDB(DB* db) { db_ = db; }
db_ = db;
}
const char* Name() const override { return "FilterWithGet"; } const char* Name() const override { return "FilterWithGet"; }
@ -358,7 +356,6 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
DB* db_; DB* db_;
}; };
std::shared_ptr<FilterWithGet> cf(new FilterWithGet()); std::shared_ptr<FilterWithGet> cf(new FilterWithGet());
Options options; Options options;
@ -385,7 +382,6 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
db->CompactFiles(ROCKSDB_NAMESPACE::CompactionOptions(), {fname}, 0)); db->CompactFiles(ROCKSDB_NAMESPACE::CompactionOptions(), {fname}, 0));
} }
delete db; delete db;
} }
@ -400,10 +396,9 @@ TEST_F(CompactFilesTest, SentinelCompressionType) {
} }
// Check that passing `CompressionType::kDisableCompressionOption` to // Check that passing `CompressionType::kDisableCompressionOption` to
// `CompactFiles` causes it to use the column family compression options. // `CompactFiles` causes it to use the column family compression options.
for (auto compaction_style : for (auto compaction_style : {CompactionStyle::kCompactionStyleLevel,
{CompactionStyle::kCompactionStyleLevel, CompactionStyle::kCompactionStyleUniversal,
CompactionStyle::kCompactionStyleUniversal, CompactionStyle::kCompactionStyleNone}) {
CompactionStyle::kCompactionStyleNone}) {
ASSERT_OK(DestroyDB(db_name_, Options())); ASSERT_OK(DestroyDB(db_name_, Options()));
Options options; Options options;
options.compaction_style = compaction_style; options.compaction_style = compaction_style;

@ -17,7 +17,6 @@
#include "util/string_util.h" #include "util/string_util.h"
#include "utilities/merge_operators.h" #include "utilities/merge_operators.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
namespace { namespace {
@ -249,7 +248,7 @@ class TwoStrComparator : public Comparator {
void FindShortSuccessor(std::string* /*key*/) const override {} void FindShortSuccessor(std::string* /*key*/) const override {}
}; };
} // namespace } // anonymous namespace
class ComparatorDBTest class ComparatorDBTest
: public testing::Test, : public testing::Test,
@ -470,7 +469,7 @@ void VerifySuccessor(const Slice& s, const Slice& t) {
ASSERT_FALSE(rbc->IsSameLengthImmediateSuccessor(t, s)); ASSERT_FALSE(rbc->IsSameLengthImmediateSuccessor(t, s));
} }
} // namespace } // anonymous namespace
TEST_P(ComparatorDBTest, IsSameLengthImmediateSuccessor) { TEST_P(ComparatorDBTest, IsSameLengthImmediateSuccessor) {
{ {

@ -26,8 +26,7 @@ Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
} }
Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family, Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family,
const RangePtr* ranges, size_t n, const RangePtr* ranges, size_t n, bool include_end) {
bool include_end) {
return (static_cast_with_check<DBImpl>(db->GetRootDB())) return (static_cast_with_check<DBImpl>(db->GetRootDB()))
->DeleteFilesInRanges(column_family, ranges, n, include_end); ->DeleteFilesInRanges(column_family, ranges, n, include_end);
} }
@ -47,9 +46,8 @@ Status VerifySstFileChecksum(const Options& options,
InternalKeyComparator internal_comparator(options.comparator); InternalKeyComparator internal_comparator(options.comparator);
ImmutableOptions ioptions(options); ImmutableOptions ioptions(options);
Status s = ioptions.fs->NewRandomAccessFile(file_path, Status s = ioptions.fs->NewRandomAccessFile(
FileOptions(env_options), file_path, FileOptions(env_options), &file, nullptr);
&file, nullptr);
if (s.ok()) { if (s.ok()) {
s = ioptions.fs->GetFileSize(file_path, IOOptions(), &file_size, nullptr); s = ioptions.fs->GetFileSize(file_path, IOOptions(), &file_size, nullptr);
} else { } else {

@ -65,7 +65,7 @@ class ErrorEnv : public EnvWrapper {
return target()->NewWritableFile(fname, result, soptions); return target()->NewWritableFile(fname, result, soptions);
} }
}; };
} // namespace } // anonymous namespace
class CorruptionTest : public testing::Test { class CorruptionTest : public testing::Test {
public: public:
std::shared_ptr<Env> env_guard_; std::shared_ptr<Env> env_guard_;
@ -138,9 +138,7 @@ class CorruptionTest : public testing::Test {
return DB::Open(opt, dbname_, &db_); return DB::Open(opt, dbname_, &db_);
} }
void Reopen(Options* options = nullptr) { void Reopen(Options* options = nullptr) { ASSERT_OK(TryReopen(options)); }
ASSERT_OK(TryReopen(options));
}
void RepairDB() { void RepairDB() {
delete db_; delete db_;
@ -156,7 +154,7 @@ class CorruptionTest : public testing::Test {
DBImpl* dbi = static_cast_with_check<DBImpl>(db_); DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
ASSERT_OK(dbi->TEST_FlushMemTable()); ASSERT_OK(dbi->TEST_FlushMemTable());
} }
//if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n); // if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
Slice key = Key(i + start, &key_space); Slice key = Key(i + start, &key_space);
batch.Clear(); batch.Clear();
ASSERT_OK(batch.Put(key, Value(i + start, &value_space))); ASSERT_OK(batch.Put(key, Value(i + start, &value_space)));
@ -183,8 +181,7 @@ class CorruptionTest : public testing::Test {
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
uint64_t key; uint64_t key;
Slice in(iter->key()); Slice in(iter->key());
if (!ConsumeDecimalNumber(&in, &key) || if (!ConsumeDecimalNumber(&in, &key) || !in.empty() ||
!in.empty() ||
key < next_expected) { key < next_expected) {
bad_keys++; bad_keys++;
continue; continue;
@ -200,10 +197,11 @@ class CorruptionTest : public testing::Test {
iter->status().PermitUncheckedError(); iter->status().PermitUncheckedError();
delete iter; delete iter;
fprintf(stderr, fprintf(
"expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%llu\n", stderr,
min_expected, max_expected, correct, bad_keys, bad_values, "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%llu\n",
static_cast<unsigned long long>(missed)); min_expected, max_expected, correct, bad_keys, bad_values,
static_cast<unsigned long long>(missed));
ASSERT_LE(min_expected, correct); ASSERT_LE(min_expected, correct);
ASSERT_GE(max_expected, correct); ASSERT_GE(max_expected, correct);
} }
@ -217,8 +215,7 @@ class CorruptionTest : public testing::Test {
std::string fname; std::string fname;
int picked_number = -1; int picked_number = -1;
for (size_t i = 0; i < filenames.size(); i++) { for (size_t i = 0; i < filenames.size(); i++) {
if (ParseFileName(filenames[i], &number, &type) && if (ParseFileName(filenames[i], &number, &type) && type == filetype &&
type == filetype &&
static_cast<int>(number) > picked_number) { // Pick latest file static_cast<int>(number) > picked_number) { // Pick latest file
fname = dbname_ + "/" + filenames[i]; fname = dbname_ + "/" + filenames[i];
picked_number = static_cast<int>(number); picked_number = static_cast<int>(number);
@ -244,7 +241,6 @@ class CorruptionTest : public testing::Test {
FAIL() << "no file found at level"; FAIL() << "no file found at level";
} }
int Property(const std::string& name) { int Property(const std::string& name) {
std::string property; std::string property;
int result; int result;

@ -77,9 +77,7 @@ class CuckooTableDBTest : public testing::Test {
return db_->Put(WriteOptions(), k, v); return db_->Put(WriteOptions(), k, v);
} }
Status Delete(const std::string& k) { Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); }
return db_->Delete(WriteOptions(), k);
}
std::string Get(const std::string& k) { std::string Get(const std::string& k) {
ReadOptions options; ReadOptions options;
@ -313,23 +311,21 @@ TEST_F(CuckooTableDBTest, AdaptiveTable) {
// Write some keys using plain table. // Write some keys using plain table.
std::shared_ptr<TableFactory> block_based_factory( std::shared_ptr<TableFactory> block_based_factory(
NewBlockBasedTableFactory()); NewBlockBasedTableFactory());
std::shared_ptr<TableFactory> plain_table_factory( std::shared_ptr<TableFactory> plain_table_factory(NewPlainTableFactory());
NewPlainTableFactory()); std::shared_ptr<TableFactory> cuckoo_table_factory(NewCuckooTableFactory());
std::shared_ptr<TableFactory> cuckoo_table_factory(
NewCuckooTableFactory());
options.create_if_missing = false; options.create_if_missing = false;
options.table_factory.reset(NewAdaptiveTableFactory( options.table_factory.reset(
plain_table_factory, block_based_factory, plain_table_factory, NewAdaptiveTableFactory(plain_table_factory, block_based_factory,
cuckoo_table_factory)); plain_table_factory, cuckoo_table_factory));
Reopen(&options); Reopen(&options);
ASSERT_OK(Put("key4", "v4")); ASSERT_OK(Put("key4", "v4"));
ASSERT_OK(Put("key1", "v5")); ASSERT_OK(Put("key1", "v5"));
ASSERT_OK(dbfull()->TEST_FlushMemTable()); ASSERT_OK(dbfull()->TEST_FlushMemTable());
// Write some keys using block based table. // Write some keys using block based table.
options.table_factory.reset(NewAdaptiveTableFactory( options.table_factory.reset(
block_based_factory, block_based_factory, plain_table_factory, NewAdaptiveTableFactory(block_based_factory, block_based_factory,
cuckoo_table_factory)); plain_table_factory, cuckoo_table_factory));
Reopen(&options); Reopen(&options);
ASSERT_OK(Put("key5", "v6")); ASSERT_OK(Put("key5", "v6"));
ASSERT_OK(Put("key2", "v7")); ASSERT_OK(Put("key2", "v7"));

@ -2805,7 +2805,7 @@ TEST_F(DBBasicTest, MultiGetIOBufferOverrun) {
table_options.pin_l0_filter_and_index_blocks_in_cache = true; table_options.pin_l0_filter_and_index_blocks_in_cache = true;
table_options.block_size = 16 * 1024; table_options.block_size = 16 * 1024;
ASSERT_TRUE(table_options.block_size > ASSERT_TRUE(table_options.block_size >
BlockBasedTable::kMultiGetReadStackBufSize); BlockBasedTable::kMultiGetReadStackBufSize);
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options); Reopen(options);
@ -2914,7 +2914,7 @@ class TableFileListener : public EventListener {
InstrumentedMutex mutex_; InstrumentedMutex mutex_;
std::unordered_map<std::string, std::vector<std::string>> cf_to_paths_; std::unordered_map<std::string, std::vector<std::string>> cf_to_paths_;
}; };
} // namespace } // anonymous namespace
TEST_F(DBBasicTest, LastSstFileNotInManifest) { TEST_F(DBBasicTest, LastSstFileNotInManifest) {
// If the last sst file is not tracked in MANIFEST, // If the last sst file is not tracked in MANIFEST,

@ -414,7 +414,7 @@ class ReadOnlyCacheWrapper : public CacheWrapper {
} }
}; };
} // namespace } // anonymous namespace
TEST_F(DBBlockCacheTest, TestWithSameCompressed) { TEST_F(DBBlockCacheTest, TestWithSameCompressed) {
auto table_options = GetTableOptions(); auto table_options = GetTableOptions();
@ -1973,7 +1973,7 @@ struct CacheKeyDecoder {
DownwardInvolution(decoded_session_counter)); DownwardInvolution(decoded_session_counter));
} }
}; };
} // namespace } // anonymous namespace
TEST_F(CacheKeyTest, Encodings) { TEST_F(CacheKeyTest, Encodings) {
// This test primarily verifies this claim from cache_key.cc: // This test primarily verifies this claim from cache_key.cc:

@ -43,7 +43,7 @@ const std::string kStandard128Ribbon =
test::Standard128RibbonFilterPolicy::kClassName(); test::Standard128RibbonFilterPolicy::kClassName();
const std::string kAutoBloom = BloomFilterPolicy::kClassName(); const std::string kAutoBloom = BloomFilterPolicy::kClassName();
const std::string kAutoRibbon = RibbonFilterPolicy::kClassName(); const std::string kAutoRibbon = RibbonFilterPolicy::kClassName();
} // namespace } // anonymous namespace
// DB tests related to bloom filter. // DB tests related to bloom filter.
@ -622,7 +622,7 @@ class AlwaysTrueFilterPolicy : public ReadOnlyBuiltinFilterPolicy {
bool skip_; bool skip_;
}; };
} // namespace } // anonymous namespace
TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) { TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
constexpr int maxKey = 10; constexpr int maxKey = 10;
@ -767,10 +767,10 @@ INSTANTIATE_TEST_CASE_P(
INSTANTIATE_TEST_CASE_P( INSTANTIATE_TEST_CASE_P(
FormatLatest, DBBloomFilterTestWithParam, FormatLatest, DBBloomFilterTestWithParam,
::testing::Values( ::testing::Values(std::make_tuple(kAutoBloom, true, kLatestFormatVersion),
std::make_tuple(kAutoBloom, true, kLatestFormatVersion), std::make_tuple(kAutoBloom, false, kLatestFormatVersion),
std::make_tuple(kAutoBloom, false, kLatestFormatVersion), std::make_tuple(kAutoRibbon, false,
std::make_tuple(kAutoRibbon, false, kLatestFormatVersion))); kLatestFormatVersion)));
#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) #endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
TEST_F(DBBloomFilterTest, BloomFilterRate) { TEST_F(DBBloomFilterTest, BloomFilterRate) {
@ -840,7 +840,7 @@ std::vector<CompatibilityConfig> kCompatibilityConfigs = {
BlockBasedTableOptions().format_version}, BlockBasedTableOptions().format_version},
{kCompatibilityRibbonPolicy, true, BlockBasedTableOptions().format_version}, {kCompatibilityRibbonPolicy, true, BlockBasedTableOptions().format_version},
}; };
} // namespace } // anonymous namespace
TEST_F(DBBloomFilterTest, BloomFilterCompatibility) { TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
Options options = CurrentOptions(); Options options = CurrentOptions();
@ -1678,7 +1678,7 @@ class TestingContextCustomFilterPolicy
private: private:
mutable std::string test_report_; mutable std::string test_report_;
}; };
} // namespace } // anonymous namespace
TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) { TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
auto policy = std::make_shared<TestingContextCustomFilterPolicy>(15, 8, 5); auto policy = std::make_shared<TestingContextCustomFilterPolicy>(15, 8, 5);
@ -2186,16 +2186,14 @@ INSTANTIATE_TEST_CASE_P(DBBloomFilterTestVaryPrefixAndFormatVer,
std::make_tuple(false, 2), std::make_tuple(false, 2),
std::make_tuple(false, 3), std::make_tuple(false, 3),
std::make_tuple(false, 4), std::make_tuple(false, 4),
std::make_tuple(false, 5), std::make_tuple(false, 5), std::make_tuple(true, 2),
std::make_tuple(true, 2), std::make_tuple(true, 3), std::make_tuple(true, 4),
std::make_tuple(true, 3),
std::make_tuple(true, 4),
std::make_tuple(true, 5))); std::make_tuple(true, 5)));
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
namespace { namespace {
static const std::string kPlainTable = "test_PlainTableBloom"; static const std::string kPlainTable = "test_PlainTableBloom";
} // namespace } // anonymous namespace
class BloomStatsTestWithParam class BloomStatsTestWithParam
: public DBBloomFilterTest, : public DBBloomFilterTest,
@ -2408,7 +2406,7 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) {
dbtest->Flush(); dbtest->Flush();
} }
} }
} // namespace } // anonymous namespace
TEST_F(DBBloomFilterTest, PrefixScan) { TEST_F(DBBloomFilterTest, PrefixScan) {
while (ChangeFilterOptions()) { while (ChangeFilterOptions()) {
@ -3169,7 +3167,7 @@ std::pair<uint64_t, uint64_t> CheckedAndUseful(uint64_t checked,
uint64_t useful) { uint64_t useful) {
return {checked, useful}; return {checked, useful};
} }
} // namespace } // anonymous namespace
// This uses a prefix_extractor + comparator combination that violates // This uses a prefix_extractor + comparator combination that violates
// one of the old obsolete, unnecessary axioms of prefix extraction: // one of the old obsolete, unnecessary axioms of prefix extraction:
@ -3377,7 +3375,7 @@ class NonIdempotentFixed4Transform : public SliceTransform {
bool InDomain(const Slice& src) const override { return src.size() >= 5; } bool InDomain(const Slice& src) const override { return src.size() >= 5; }
}; };
} // namespace } // anonymous namespace
// This uses a prefix_extractor + comparator combination that violates // This uses a prefix_extractor + comparator combination that violates
// two of the old obsolete, unnecessary axioms of prefix extraction: // two of the old obsolete, unnecessary axioms of prefix extraction:

@ -255,9 +255,8 @@ Options DeletionTriggerOptions(Options options) {
return options; return options;
} }
bool HaveOverlappingKeyRanges( bool HaveOverlappingKeyRanges(const Comparator* c, const SstFileMetaData& a,
const Comparator* c, const SstFileMetaData& b) {
const SstFileMetaData& a, const SstFileMetaData& b) {
if (c->CompareWithoutTimestamp(a.smallestkey, b.smallestkey) >= 0) { if (c->CompareWithoutTimestamp(a.smallestkey, b.smallestkey) >= 0) {
if (c->CompareWithoutTimestamp(a.smallestkey, b.largestkey) <= 0) { if (c->CompareWithoutTimestamp(a.smallestkey, b.largestkey) <= 0) {
// b.smallestkey <= a.smallestkey <= b.largestkey // b.smallestkey <= a.smallestkey <= b.largestkey
@ -282,18 +281,15 @@ bool HaveOverlappingKeyRanges(
// Identifies all files between level "min_level" and "max_level" // Identifies all files between level "min_level" and "max_level"
// which has overlapping key range with "input_file_meta". // which has overlapping key range with "input_file_meta".
void GetOverlappingFileNumbersForLevelCompaction( void GetOverlappingFileNumbersForLevelCompaction(
const ColumnFamilyMetaData& cf_meta, const ColumnFamilyMetaData& cf_meta, const Comparator* comparator,
const Comparator* comparator, int min_level, int max_level, const SstFileMetaData* input_file_meta,
int min_level, int max_level,
const SstFileMetaData* input_file_meta,
std::set<std::string>* overlapping_file_names) { std::set<std::string>* overlapping_file_names) {
std::set<const SstFileMetaData*> overlapping_files; std::set<const SstFileMetaData*> overlapping_files;
overlapping_files.insert(input_file_meta); overlapping_files.insert(input_file_meta);
for (int m = min_level; m <= max_level; ++m) { for (int m = min_level; m <= max_level; ++m) {
for (auto& file : cf_meta.levels[m].files) { for (auto& file : cf_meta.levels[m].files) {
for (auto* included_file : overlapping_files) { for (auto* included_file : overlapping_files) {
if (HaveOverlappingKeyRanges( if (HaveOverlappingKeyRanges(comparator, *included_file, file)) {
comparator, *included_file, file)) {
overlapping_files.insert(&file); overlapping_files.insert(&file);
overlapping_file_names->insert(file.name); overlapping_file_names->insert(file.name);
break; break;
@ -316,12 +312,9 @@ void VerifyCompactionResult(
#endif #endif
} }
const SstFileMetaData* PickFileRandomly( const SstFileMetaData* PickFileRandomly(const ColumnFamilyMetaData& cf_meta,
const ColumnFamilyMetaData& cf_meta, Random* rand, int* level = nullptr) {
Random* rand, auto file_id = rand->Uniform(static_cast<int>(cf_meta.file_count)) + 1;
int* level = nullptr) {
auto file_id = rand->Uniform(static_cast<int>(
cf_meta.file_count)) + 1;
for (auto& level_meta : cf_meta.levels) { for (auto& level_meta : cf_meta.levels) {
if (file_id <= level_meta.files.size()) { if (file_id <= level_meta.files.size()) {
if (level != nullptr) { if (level != nullptr) {
@ -747,7 +740,6 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
} }
} }
TEST_P(DBCompactionTestWithParam, CompactionTrigger) { TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
const int kNumKeysPerFile = 100; const int kNumKeysPerFile = 100;
@ -890,7 +882,7 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) { TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.write_buffer_size = 100000000; // Large write buffer options.write_buffer_size = 100000000; // Large write buffer
options.max_subcompactions = max_subcompactions_; options.max_subcompactions = max_subcompactions_;
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -1076,7 +1068,7 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
compact_opt.compression = kNoCompression; compact_opt.compression = kNoCompression;
compact_opt.output_file_size_limit = 4096; compact_opt.output_file_size_limit = 4096;
const size_t key_len = const size_t key_len =
static_cast<size_t>(compact_opt.output_file_size_limit) / 5; static_cast<size_t>(compact_opt.output_file_size_limit) / 5;
DestroyAndReopen(options); DestroyAndReopen(options);
@ -1254,14 +1246,8 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveNonOverlappingFiles) {
DestroyAndReopen(options); DestroyAndReopen(options);
// non overlapping ranges // non overlapping ranges
std::vector<std::pair<int32_t, int32_t>> ranges = { std::vector<std::pair<int32_t, int32_t>> ranges = {
{100, 199}, {100, 199}, {300, 399}, {0, 99}, {200, 299},
{300, 399}, {600, 699}, {400, 499}, {500, 550}, {551, 599},
{0, 99},
{200, 299},
{600, 699},
{400, 499},
{500, 550},
{551, 599},
}; };
int32_t value_size = 10 * 1024; // 10 KB int32_t value_size = 10 * 1024; // 10 KB
@ -1304,14 +1290,15 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveNonOverlappingFiles) {
DestroyAndReopen(options); DestroyAndReopen(options);
// Same ranges as above but overlapping // Same ranges as above but overlapping
ranges = { ranges = {
{100, 199}, {100, 199},
{300, 399}, {300, 399},
{0, 99}, {0, 99},
{200, 299}, {200, 299},
{600, 699}, {600, 699},
{400, 499}, {400, 499},
{500, 560}, // this range overlap with the next one {500, 560}, // this range overlap with the next
{551, 599}, // one
{551, 599},
}; };
for (size_t i = 0; i < ranges.size(); i++) { for (size_t i = 0; i < ranges.size(); i++) {
for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) { for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) {
@ -1914,7 +1901,7 @@ TEST_F(DBCompactionTest, DeleteFilesInRanges) {
ASSERT_EQ("0,0,10", FilesPerLevel(0)); ASSERT_EQ("0,0,10", FilesPerLevel(0));
// file [0 => 100), [200 => 300), ... [800, 900) // file [0 => 100), [200 => 300), ... [800, 900)
for (auto i = 0; i < 10; i+=2) { for (auto i = 0; i < 10; i += 2) {
for (auto j = 0; j < 100; j++) { for (auto j = 0; j < 100; j++) {
auto k = i * 100 + j; auto k = i * 100 + j;
ASSERT_OK(Put(Key(k), values[k])); ASSERT_OK(Put(Key(k), values[k]));
@ -2356,14 +2343,14 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionCFPathUse) {
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_2", 4 * 1024 * 1024); cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_2", 4 * 1024 * 1024);
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 1024 * 1024 * 1024); cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 1024 * 1024 * 1024);
option_vector.emplace_back(DBOptions(options), cf_opt1); option_vector.emplace_back(DBOptions(options), cf_opt1);
CreateColumnFamilies({"one"},option_vector[1]); CreateColumnFamilies({"one"}, option_vector[1]);
// Configure CF2 specific paths. // Configure CF2 specific paths.
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 500 * 1024); cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 500 * 1024);
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_2", 4 * 1024 * 1024); cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_2", 4 * 1024 * 1024);
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 1024 * 1024 * 1024); cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 1024 * 1024 * 1024);
option_vector.emplace_back(DBOptions(options), cf_opt2); option_vector.emplace_back(DBOptions(options), cf_opt2);
CreateColumnFamilies({"two"},option_vector[2]); CreateColumnFamilies({"two"}, option_vector[2]);
ReopenWithColumnFamilies({"default", "one", "two"}, option_vector); ReopenWithColumnFamilies({"default", "one", "two"}, option_vector);
@ -2736,7 +2723,6 @@ TEST_P(DBCompactionTestWithParam, ManualCompaction) {
} }
} }
TEST_P(DBCompactionTestWithParam, ManualLevelCompactionOutputPathId) { TEST_P(DBCompactionTestWithParam, ManualLevelCompactionOutputPathId) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.db_paths.emplace_back(dbname_ + "_2", 2 * 10485760); options.db_paths.emplace_back(dbname_ + "_2", 2 * 10485760);
@ -2873,14 +2859,13 @@ TEST_P(DBCompactionTestWithParam, DISABLED_CompactFilesOnLevelCompaction) {
auto file_meta = PickFileRandomly(cf_meta, &rnd, &level); auto file_meta = PickFileRandomly(cf_meta, &rnd, &level);
compaction_input_file_names.push_back(file_meta->name); compaction_input_file_names.push_back(file_meta->name);
GetOverlappingFileNumbersForLevelCompaction( GetOverlappingFileNumbersForLevelCompaction(
cf_meta, options.comparator, level, output_level, cf_meta, options.comparator, level, output_level, file_meta,
file_meta, &overlapping_file_names); &overlapping_file_names);
} }
ASSERT_OK(dbfull()->CompactFiles( ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
CompactionOptions(), handles_[1], compaction_input_file_names,
compaction_input_file_names, output_level));
output_level));
// Make sure all overlapping files do not exist after compaction // Make sure all overlapping files do not exist after compaction
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta); dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
@ -2903,8 +2888,7 @@ TEST_P(DBCompactionTestWithParam, PartialCompactionFailure) {
options.write_buffer_size = kKeysPerBuffer * kKvSize; options.write_buffer_size = kKeysPerBuffer * kKvSize;
options.max_write_buffer_number = 2; options.max_write_buffer_number = 2;
options.target_file_size_base = options.target_file_size_base =
options.write_buffer_size * options.write_buffer_size * (options.max_write_buffer_number - 1);
(options.max_write_buffer_number - 1);
options.level0_file_num_compaction_trigger = kNumL1Files; options.level0_file_num_compaction_trigger = kNumL1Files;
options.max_bytes_for_level_base = options.max_bytes_for_level_base =
options.level0_file_num_compaction_trigger * options.level0_file_num_compaction_trigger *
@ -2924,10 +2908,9 @@ TEST_P(DBCompactionTestWithParam, PartialCompactionFailure) {
DestroyAndReopen(options); DestroyAndReopen(options);
const int kNumInsertedKeys = const int kNumInsertedKeys = options.level0_file_num_compaction_trigger *
options.level0_file_num_compaction_trigger * (options.max_write_buffer_number - 1) *
(options.max_write_buffer_number - 1) * kKeysPerBuffer;
kKeysPerBuffer;
Random rnd(301); Random rnd(301);
std::vector<std::string> keys; std::vector<std::string> keys;
@ -3625,9 +3608,8 @@ TEST_F(DBCompactionTest, CompactFilesPendingL0Bug) {
ASSERT_EQ(kNumL0Files, cf_meta.levels[0].files.size()); ASSERT_EQ(kNumL0Files, cf_meta.levels[0].files.size());
std::vector<std::string> input_filenames; std::vector<std::string> input_filenames;
input_filenames.push_back(cf_meta.levels[0].files.front().name); input_filenames.push_back(cf_meta.levels[0].files.front().name);
ASSERT_OK(dbfull() ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), input_filenames,
->CompactFiles(CompactionOptions(), input_filenames, 0 /* output_level */));
0 /* output_level */));
TEST_SYNC_POINT("DBCompactionTest::CompactFilesPendingL0Bug:ManualCompacted"); TEST_SYNC_POINT("DBCompactionTest::CompactFilesPendingL0Bug:ManualCompacted");
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
} }
@ -4510,9 +4492,9 @@ TEST_F(DBCompactionTest, LevelPeriodicAndTtlCompaction) {
const int kValueSize = 100; const int kValueSize = 100;
Options options = CurrentOptions(); Options options = CurrentOptions();
options.ttl = 10 * 60 * 60; // 10 hours options.ttl = 10 * 60 * 60; // 10 hours
options.periodic_compaction_seconds = 48 * 60 * 60; // 2 days options.periodic_compaction_seconds = 48 * 60 * 60; // 2 days
options.max_open_files = -1; // needed for both periodic and ttl compactions options.max_open_files = -1; // needed for both periodic and ttl compactions
env_->SetMockSleep(); env_->SetMockSleep();
options.env = env_; options.env = env_;
@ -4934,7 +4916,7 @@ TEST_F(DBCompactionTest, CompactRangeSkipFlushAfterDelay) {
{"DBImpl::FlushMemTable:StallWaitDone", "CompactionJob::Run():End"}}); {"DBImpl::FlushMemTable:StallWaitDone", "CompactionJob::Run():End"}});
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
//used for the delayable flushes // used for the delayable flushes
FlushOptions flush_opts; FlushOptions flush_opts;
flush_opts.allow_write_stall = true; flush_opts.allow_write_stall = true;
for (int i = 0; i < kNumL0FilesLimit - 1; ++i) { for (int i = 0; i < kNumL0FilesLimit - 1; ++i) {
@ -4953,7 +4935,8 @@ TEST_F(DBCompactionTest, CompactRangeSkipFlushAfterDelay) {
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024))); ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
ASSERT_OK(dbfull()->Flush(flush_opts)); ASSERT_OK(dbfull()->Flush(flush_opts));
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024))); ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
TEST_SYNC_POINT("DBCompactionTest::CompactRangeSkipFlushAfterDelay:PostFlush"); TEST_SYNC_POINT(
"DBCompactionTest::CompactRangeSkipFlushAfterDelay:PostFlush");
manual_compaction_thread.join(); manual_compaction_thread.join();
// If CompactRange's flush was skipped, the final Put above will still be // If CompactRange's flush was skipped, the final Put above will still be
@ -5246,10 +5229,10 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
} }
std::shared_ptr<ConcurrentTaskLimiter> unique_limiter( std::shared_ptr<ConcurrentTaskLimiter> unique_limiter(
NewConcurrentTaskLimiter("unique_limiter", -1)); NewConcurrentTaskLimiter("unique_limiter", -1));
const char* cf_names[] = {"default", "0", "1", "2", "3", "4", "5", const char* cf_names[] = {"default", "0", "1", "2", "3", "4", "5", "6", "7",
"6", "7", "8", "9", "a", "b", "c", "d", "e", "f" }; "8", "9", "a", "b", "c", "d", "e", "f"};
const unsigned int cf_count = sizeof cf_names / sizeof cf_names[0]; const unsigned int cf_count = sizeof cf_names / sizeof cf_names[0];
std::unordered_map<std::string, CompactionLimiter*> cf_to_limiter; std::unordered_map<std::string, CompactionLimiter*> cf_to_limiter;
@ -5261,10 +5244,10 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
options.level0_file_num_compaction_trigger = 4; options.level0_file_num_compaction_trigger = 4;
options.level0_slowdown_writes_trigger = 64; options.level0_slowdown_writes_trigger = 64;
options.level0_stop_writes_trigger = 64; options.level0_stop_writes_trigger = 64;
options.max_background_jobs = kMaxBackgroundThreads; // Enough threads options.max_background_jobs = kMaxBackgroundThreads; // Enough threads
options.memtable_factory.reset( options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile)); test::NewSpecialSkipListFactory(kNumKeysPerFile));
options.max_write_buffer_number = 10; // Enough memtables options.max_write_buffer_number = 10; // Enough memtables
DestroyAndReopen(options); DestroyAndReopen(options);
std::vector<Options> option_vector; std::vector<Options> option_vector;
@ -5292,9 +5275,8 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
CreateColumnFamilies({cf_names[cf]}, option_vector[cf]); CreateColumnFamilies({cf_names[cf]}, option_vector[cf]);
} }
ReopenWithColumnFamilies(std::vector<std::string>(cf_names, ReopenWithColumnFamilies(
cf_names + cf_count), std::vector<std::string>(cf_names, cf_names + cf_count), option_vector);
option_vector);
port::Mutex mutex; port::Mutex mutex;
@ -5356,7 +5338,7 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
// Enough L0 files to trigger compaction // Enough L0 files to trigger compaction
for (unsigned int cf = 0; cf < cf_count; cf++) { for (unsigned int cf = 0; cf < cf_count; cf++) {
ASSERT_EQ(NumTableFilesAtLevel(0, cf), ASSERT_EQ(NumTableFilesAtLevel(0, cf),
options.level0_file_num_compaction_trigger); options.level0_file_num_compaction_trigger);
} }
// Create more files for one column family, which triggers speed up // Create more files for one column family, which triggers speed up
@ -5399,7 +5381,7 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
// flush one more file to cf 1 // flush one more file to cf 1
for (int i = 0; i < kNumKeysPerFile; i++) { for (int i = 0; i < kNumKeysPerFile; i++) {
ASSERT_OK(Put(cf_test, Key(keyIndex++), "")); ASSERT_OK(Put(cf_test, Key(keyIndex++), ""));
} }
// put extra key to trigger flush // put extra key to trigger flush
ASSERT_OK(Put(cf_test, "", "")); ASSERT_OK(Put(cf_test, "", ""));
@ -5434,9 +5416,7 @@ TEST_P(DBCompactionDirectIOTest, DirectIO) {
}); });
if (options.use_direct_io_for_flush_and_compaction) { if (options.use_direct_io_for_flush_and_compaction) {
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
"SanitizeOptions:direct_io", [&](void* /*arg*/) { "SanitizeOptions:direct_io", [&](void* /*arg*/) { readahead = true; });
readahead = true;
});
} }
SyncPoint::GetInstance()->EnableProcessing(); SyncPoint::GetInstance()->EnableProcessing();
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -8404,8 +8384,8 @@ int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
#else #else
(void) argc; (void)argc;
(void) argv; (void)argv;
return 0; return 0;
#endif #endif
} }

@ -500,8 +500,8 @@ int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
#else #else
(void) argc; (void)argc;
(void) argv; (void)argv;
return 0; return 0;
#endif #endif
} }

@ -43,7 +43,7 @@ TEST_F(DBEncryptionTest, CheckEncrypted) {
Env* target = GetTargetEnv(); Env* target = GetTargetEnv();
int hits = 0; int hits = 0;
for (auto it = fileNames.begin() ; it != fileNames.end(); ++it) { for (auto it = fileNames.begin(); it != fileNames.end(); ++it) {
if (*it == "LOCK") { if (*it == "LOCK") {
continue; continue;
} }
@ -64,24 +64,24 @@ TEST_F(DBEncryptionTest, CheckEncrypted) {
ASSERT_OK(status); ASSERT_OK(status);
if (data.ToString().find("foo567") != std::string::npos) { if (data.ToString().find("foo567") != std::string::npos) {
hits++; hits++;
//std::cout << "Hit in " << filePath << "\n"; // std::cout << "Hit in " << filePath << "\n";
} }
if (data.ToString().find("v1.fetdq") != std::string::npos) { if (data.ToString().find("v1.fetdq") != std::string::npos) {
hits++; hits++;
//std::cout << "Hit in " << filePath << "\n"; // std::cout << "Hit in " << filePath << "\n";
} }
if (data.ToString().find("bar123") != std::string::npos) { if (data.ToString().find("bar123") != std::string::npos) {
hits++; hits++;
//std::cout << "Hit in " << filePath << "\n"; // std::cout << "Hit in " << filePath << "\n";
} }
if (data.ToString().find("v2.dfgkjdfghsd") != std::string::npos) { if (data.ToString().find("v2.dfgkjdfghsd") != std::string::npos) {
hits++; hits++;
//std::cout << "Hit in " << filePath << "\n"; // std::cout << "Hit in " << filePath << "\n";
} }
if (data.ToString().find("dfgk") != std::string::npos) { if (data.ToString().find("dfgk") != std::string::npos) {
hits++; hits++;
//std::cout << "Hit in " << filePath << "\n"; // std::cout << "Hit in " << filePath << "\n";
} }
} }
if (encrypted_env_) { if (encrypted_env_) {
@ -119,7 +119,7 @@ TEST_F(DBEncryptionTest, ReadEmptyFile) {
ASSERT_TRUE(data.empty()); ASSERT_TRUE(data.empty());
} }
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -65,8 +65,7 @@ Status DBImpl::FlushForGetLiveFiles() {
} }
Status DBImpl::GetLiveFiles(std::vector<std::string>& ret, Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
uint64_t* manifest_file_size, uint64_t* manifest_file_size, bool flush_memtable) {
bool flush_memtable) {
*manifest_file_size = 0; *manifest_file_size = 0;
mutex_.Lock(); mutex_.Lock();

@ -57,7 +57,7 @@ TEST_F(DBFlushTest, FlushWhileWritingManifest) {
Reopen(options); Reopen(options);
FlushOptions no_wait; FlushOptions no_wait;
no_wait.wait = false; no_wait.wait = false;
no_wait.allow_write_stall=true; no_wait.allow_write_stall = true;
SyncPoint::GetInstance()->LoadDependency( SyncPoint::GetInstance()->LoadDependency(
{{"VersionSet::LogAndApply:WriteManifest", {{"VersionSet::LogAndApply:WriteManifest",
@ -1822,8 +1822,8 @@ TEST_F(DBFlushTest, ManualFlushFailsInReadOnlyMode) {
ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable()); ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable());
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
uint64_t num_bg_errors; uint64_t num_bg_errors;
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBackgroundErrors, ASSERT_TRUE(
&num_bg_errors)); db_->GetIntProperty(DB::Properties::kBackgroundErrors, &num_bg_errors));
ASSERT_GT(num_bg_errors, 0); ASSERT_GT(num_bg_errors, 0);
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE

@ -6,6 +6,7 @@
#include "db/db_info_dumper.h" #include "db/db_info_dumper.h"
#include <stdio.h> #include <stdio.h>
#include <algorithm> #include <algorithm>
#include <cinttypes> #include <cinttypes>
#include <string> #include <string>

@ -392,7 +392,7 @@ struct ReferenceIterator {
} }
}; };
} // namespace } // anonymous namespace
// Use an internal iterator that sometimes returns errors and sometimes // Use an internal iterator that sometimes returns errors and sometimes
// adds/removes entries on the fly. Do random operations on a DBIter and // adds/removes entries on the fly. Do random operations on a DBIter and
@ -482,12 +482,11 @@ TEST_F(DBIteratorStressTest, StressTest) {
std::cout << "entries:"; std::cout << "entries:";
for (size_t i = 0; i < data.entries.size(); ++i) { for (size_t i = 0; i < data.entries.size(); ++i) {
Entry& e = data.entries[i]; Entry& e = data.entries[i];
std::cout std::cout << "\n idx " << i << ": \"" << e.key << "\": \""
<< "\n idx " << i << ": \"" << e.key << "\": \"" << e.value << "\" seq: " << e.sequence << " type: "
<< e.value << "\" seq: " << e.sequence << " type: " << (e.type == kTypeValue ? "val"
<< (e.type == kTypeValue : e.type == kTypeDeletion ? "del"
? "val" : "merge");
: e.type == kTypeDeletion ? "del" : "merge");
} }
std::cout << std::endl; std::cout << std::endl;
} }

@ -3,12 +3,13 @@
// COPYING file in the root directory) and Apache 2.0 License // COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#include <string> #include "db/db_iter.h"
#include <vector>
#include <algorithm> #include <algorithm>
#include <string>
#include <utility> #include <utility>
#include <vector>
#include "db/db_iter.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "rocksdb/comparator.h" #include "rocksdb/comparator.h"
#include "rocksdb/options.h" #include "rocksdb/options.h"
@ -82,8 +83,8 @@ class TestIterator : public InternalIterator {
std::sort(data_.begin(), data_.end(), std::sort(data_.begin(), data_.end(),
[this](std::pair<std::string, std::string> a, [this](std::pair<std::string, std::string> a,
std::pair<std::string, std::string> b) { std::pair<std::string, std::string> b) {
return (cmp.Compare(a.first, b.first) < 0); return (cmp.Compare(a.first, b.first) < 0);
}); });
} }
// Removes the key from the set of keys over which this iterator iterates. // Removes the key from the set of keys over which this iterator iterates.
@ -429,7 +430,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
db_iter->SeekToLast(); db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid()); ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_key_skipped_count), 1); ASSERT_EQ(static_cast<int>(get_perf_context()->internal_key_skipped_count),
1);
ASSERT_EQ(db_iter->key().ToString(), "b"); ASSERT_EQ(db_iter->key().ToString(), "b");
SetPerfLevel(kDisable); SetPerfLevel(kDisable);
@ -557,7 +559,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
db_iter->SeekToLast(); db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid()); ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0); ASSERT_EQ(
static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
ASSERT_EQ(db_iter->key().ToString(), "b"); ASSERT_EQ(db_iter->key().ToString(), "b");
SetPerfLevel(kDisable); SetPerfLevel(kDisable);
@ -3013,7 +3016,6 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace8) {
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
} }
TEST_F(DBIteratorTest, SeekPrefixTombstones) { TEST_F(DBIteratorTest, SeekPrefixTombstones) {
ReadOptions ro; ReadOptions ro;
Options options; Options options;

@ -236,7 +236,7 @@ namespace {
std::string MakeLongKey(size_t length, char c) { std::string MakeLongKey(size_t length, char c) {
return std::string(length, c); return std::string(length, c);
} }
} // namespace } // anonymous namespace
TEST_P(DBIteratorTest, IterLongKeys) { TEST_P(DBIteratorTest, IterLongKeys) {
ASSERT_OK(Put(MakeLongKey(20, 0), "0")); ASSERT_OK(Put(MakeLongKey(20, 0), "0"));
@ -1037,7 +1037,8 @@ TEST_P(DBIteratorTest, DBIteratorBoundTest) {
iter->Next(); iter->Next();
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 2); ASSERT_EQ(
static_cast<int>(get_perf_context()->internal_delete_skipped_count), 2);
// now testing with iterate_bound // now testing with iterate_bound
Slice prefix("c"); Slice prefix("c");
@ -1060,7 +1061,8 @@ TEST_P(DBIteratorTest, DBIteratorBoundTest) {
// even though the key is deleted // even though the key is deleted
// hence internal_delete_skipped_count should be 0 // hence internal_delete_skipped_count should be 0
ASSERT_TRUE(!iter->Valid()); ASSERT_TRUE(!iter->Valid());
ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0); ASSERT_EQ(
static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
} }
} }
@ -1536,7 +1538,7 @@ class DBIteratorTestForPinnedData : public DBIteratorTest {
} }
delete iter; delete iter;
} }
}; };
#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN) #if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
@ -2180,8 +2182,8 @@ TEST_P(DBIteratorTest, IteratorWithLocalStatistics) {
ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV), (uint64_t)total_prev); ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV), (uint64_t)total_prev);
ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV_FOUND), ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV_FOUND),
(uint64_t)total_prev_found); (uint64_t)total_prev_found);
ASSERT_EQ(TestGetTickerCount(options, ITER_BYTES_READ), (uint64_t)total_bytes); ASSERT_EQ(TestGetTickerCount(options, ITER_BYTES_READ),
(uint64_t)total_bytes);
} }
TEST_P(DBIteratorTest, ReadAhead) { TEST_P(DBIteratorTest, ReadAhead) {
@ -2310,8 +2312,8 @@ TEST_P(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
EXPECT_EQ(get_perf_context()->internal_merge_count, 0); EXPECT_EQ(get_perf_context()->internal_merge_count, 0);
EXPECT_GE(get_perf_context()->internal_recent_skipped_count, 2); EXPECT_GE(get_perf_context()->internal_recent_skipped_count, 2);
EXPECT_GE(get_perf_context()->seek_on_memtable_count, 2); EXPECT_GE(get_perf_context()->seek_on_memtable_count, 2);
EXPECT_EQ(1, options.statistics->getTickerCount( EXPECT_EQ(1,
NUMBER_OF_RESEEKS_IN_ITERATION)); options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION));
} }
TEST_P(DBIteratorTest, Refresh) { TEST_P(DBIteratorTest, Refresh) {
@ -2592,7 +2594,7 @@ TEST_P(DBIteratorTest, SkipStatistics) {
} }
ASSERT_EQ(count, 3); ASSERT_EQ(count, 3);
delete iter; delete iter;
skip_count += 8; // 3 deletes + 3 original keys + 2 lower in sequence skip_count += 8; // 3 deletes + 3 original keys + 2 lower in sequence
ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP)); ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
iter = NewIterator(ReadOptions()); iter = NewIterator(ReadOptions());
@ -2603,7 +2605,7 @@ TEST_P(DBIteratorTest, SkipStatistics) {
} }
ASSERT_EQ(count, 3); ASSERT_EQ(count, 3);
delete iter; delete iter;
skip_count += 8; // Same as above, but in reverse order skip_count += 8; // Same as above, but in reverse order
ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP)); ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
ASSERT_OK(Put("aa", "1")); ASSERT_OK(Put("aa", "1"));
@ -2621,18 +2623,18 @@ TEST_P(DBIteratorTest, SkipStatistics) {
iter = NewIterator(ro); iter = NewIterator(ro);
count = 0; count = 0;
for(iter->Seek("aa"); iter->Valid(); iter->Next()) { for (iter->Seek("aa"); iter->Valid(); iter->Next()) {
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
count++; count++;
} }
ASSERT_EQ(count, 1); ASSERT_EQ(count, 1);
delete iter; delete iter;
skip_count += 6; // 3 deletes + 3 original keys skip_count += 6; // 3 deletes + 3 original keys
ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP)); ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
iter = NewIterator(ro); iter = NewIterator(ro);
count = 0; count = 0;
for(iter->SeekToLast(); iter->Valid(); iter->Prev()) { for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
count++; count++;
} }

@ -55,14 +55,13 @@ SequenceNumber ReadRecords(std::unique_ptr<TransactionLogIterator>& iter,
return res.sequence; return res.sequence;
} }
void ExpectRecords( void ExpectRecords(const int expected_no_records,
const int expected_no_records, std::unique_ptr<TransactionLogIterator>& iter) {
std::unique_ptr<TransactionLogIterator>& iter) {
int num_records; int num_records;
ReadRecords(iter, num_records); ReadRecords(iter, num_records);
ASSERT_EQ(num_records, expected_no_records); ASSERT_EQ(num_records, expected_no_records);
} }
} // namespace } // anonymous namespace
TEST_F(DBTestXactLogIterator, TransactionLogIterator) { TEST_F(DBTestXactLogIterator, TransactionLogIterator) {
do { do {
@ -95,10 +94,9 @@ TEST_F(DBTestXactLogIterator, TransactionLogIterator) {
TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) { TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) {
static const int LOG_ITERATOR_RACE_TEST_COUNT = 2; static const int LOG_ITERATOR_RACE_TEST_COUNT = 2;
static const char* sync_points[LOG_ITERATOR_RACE_TEST_COUNT][4] = { static const char* sync_points[LOG_ITERATOR_RACE_TEST_COUNT][4] = {
{"WalManager::GetSortedWalFiles:1", "WalManager::PurgeObsoleteFiles:1", {"WalManager::GetSortedWalFiles:1", "WalManager::PurgeObsoleteFiles:1",
"WalManager::PurgeObsoleteFiles:2", "WalManager::GetSortedWalFiles:2"}, "WalManager::PurgeObsoleteFiles:2", "WalManager::GetSortedWalFiles:2"},
{"WalManager::GetSortedWalsOfType:1", {"WalManager::GetSortedWalsOfType:1", "WalManager::PurgeObsoleteFiles:1",
"WalManager::PurgeObsoleteFiles:1",
"WalManager::PurgeObsoleteFiles:2", "WalManager::PurgeObsoleteFiles:2",
"WalManager::GetSortedWalsOfType:2"}}; "WalManager::GetSortedWalsOfType:2"}};
for (int test = 0; test < LOG_ITERATOR_RACE_TEST_COUNT; ++test) { for (int test = 0; test < LOG_ITERATOR_RACE_TEST_COUNT; ++test) {
@ -300,8 +298,8 @@ int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
#else #else
(void) argc; (void)argc;
(void) argv; (void)argv;
return 0; return 0;
#endif #endif
} }

@ -224,8 +224,8 @@ TEST_F(DBLogicalBlockSizeCacheTest, CreateColumnFamilies) {
// Now cf_path_0_ in cache_ has been properly decreased and cf_path_0_'s entry // Now cf_path_0_ in cache_ has been properly decreased and cf_path_0_'s entry
// is dropped from cache // is dropped from cache
ASSERT_EQ(0, cache_->Size()); ASSERT_EQ(0, cache_->Size());
ASSERT_OK(DestroyDB(dbname_, options, ASSERT_OK(
{{"cf1", cf_options}, {"cf2", cf_options}})); DestroyDB(dbname_, options, {{"cf1", cf_options}, {"cf2", cf_options}}));
} }
TEST_F(DBLogicalBlockSizeCacheTest, OpenWithColumnFamilies) { TEST_F(DBLogicalBlockSizeCacheTest, OpenWithColumnFamilies) {
@ -313,8 +313,8 @@ TEST_F(DBLogicalBlockSizeCacheTest, OpenWithColumnFamilies) {
delete db; delete db;
ASSERT_EQ(0, cache_->Size()); ASSERT_EQ(0, cache_->Size());
} }
ASSERT_OK(DestroyDB(dbname_, options, ASSERT_OK(
{{"cf1", cf_options}, {"cf2", cf_options}})); DestroyDB(dbname_, options, {{"cf1", cf_options}, {"cf2", cf_options}}));
} }
TEST_F(DBLogicalBlockSizeCacheTest, DestroyColumnFamilyHandle) { TEST_F(DBLogicalBlockSizeCacheTest, DestroyColumnFamilyHandle) {

@ -39,7 +39,7 @@ class LimitedStringAppendMergeOp : public StringAppendTESTOperator {
private: private:
size_t limit_ = 0; size_t limit_ = 0;
}; };
} // namespace } // anonymous namespace
class DBMergeOperandTest : public DBTestBase { class DBMergeOperandTest : public DBTestBase {
public: public:

@ -84,8 +84,7 @@ TEST_F(DBMergeOperatorTest, LimitMergeOperands) {
Options options; Options options;
options.create_if_missing = true; options.create_if_missing = true;
// Use only the latest two merge operands. // Use only the latest two merge operands.
options.merge_operator = options.merge_operator = std::make_shared<LimitedStringAppendMergeOp>(2, ',');
std::make_shared<LimitedStringAppendMergeOp>(2, ',');
options.env = env_; options.env = env_;
Reopen(options); Reopen(options);
// All K1 values are in memtable. // All K1 values are in memtable.
@ -203,7 +202,6 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnIteration) {
VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}}); VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}});
} }
class MergeOperatorPinningTest : public DBMergeOperatorTest, class MergeOperatorPinningTest : public DBMergeOperatorTest,
public testing::WithParamInterface<bool> { public testing::WithParamInterface<bool> {
public: public:
@ -471,7 +469,7 @@ TEST_F(DBMergeOperatorTest, TailingIteratorMemtableUnrefedBySomeoneElse) {
"DBIter::MergeValuesNewToOld:SteppedToNextOperand", [&](void*) { "DBIter::MergeValuesNewToOld:SteppedToNextOperand", [&](void*) {
EXPECT_FALSE(stepped_to_next_operand); EXPECT_FALSE(stepped_to_next_operand);
stepped_to_next_operand = true; stepped_to_next_operand = true;
someone_else.reset(); // Unpin SuperVersion A someone_else.reset(); // Unpin SuperVersion A
}); });
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();

@ -402,7 +402,7 @@ TEST_F(DBOptionsTest, SetWalBytesPerSync) {
// Do not flush. If we flush here, SwitchWAL will reuse old WAL file since its // Do not flush. If we flush here, SwitchWAL will reuse old WAL file since its
// empty and will not get the new wal_bytes_per_sync value. // empty and will not get the new wal_bytes_per_sync value.
low_bytes_per_sync = counter; low_bytes_per_sync = counter;
//5242880 = 1024 * 1024 * 5 // 5242880 = 1024 * 1024 * 5
ASSERT_OK(dbfull()->SetDBOptions({{"wal_bytes_per_sync", "5242880"}})); ASSERT_OK(dbfull()->SetDBOptions({{"wal_bytes_per_sync", "5242880"}}));
ASSERT_EQ(5242880, dbfull()->GetDBOptions().wal_bytes_per_sync); ASSERT_EQ(5242880, dbfull()->GetDBOptions().wal_bytes_per_sync);
counter = 0; counter = 0;
@ -604,7 +604,7 @@ TEST_F(DBOptionsTest, SetOptionsMayTriggerCompaction) {
TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) { TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) {
Options options; Options options;
options.create_if_missing = true; options.create_if_missing = true;
options.max_background_compactions = 1; // default value options.max_background_compactions = 1; // default value
options.env = env_; options.env = env_;
Reopen(options); Reopen(options);
ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
@ -627,7 +627,6 @@ TEST_F(DBOptionsTest, SetBackgroundFlushThreads) {
ASSERT_EQ(3, dbfull()->TEST_BGFlushesAllowed()); ASSERT_EQ(3, dbfull()->TEST_BGFlushesAllowed());
} }
TEST_F(DBOptionsTest, SetBackgroundJobs) { TEST_F(DBOptionsTest, SetBackgroundJobs) {
Options options; Options options;
options.create_if_missing = true; options.create_if_missing = true;
@ -691,7 +690,8 @@ TEST_F(DBOptionsTest, SetDelayedWriteRateOption) {
options.delayed_write_rate = 2 * 1024U * 1024U; options.delayed_write_rate = 2 * 1024U * 1024U;
options.env = env_; options.env = env_;
Reopen(options); Reopen(options);
ASSERT_EQ(2 * 1024U * 1024U, dbfull()->TEST_write_controler().max_delayed_write_rate()); ASSERT_EQ(2 * 1024U * 1024U,
dbfull()->TEST_write_controler().max_delayed_write_rate());
ASSERT_OK(dbfull()->SetDBOptions({{"delayed_write_rate", "20000"}})); ASSERT_OK(dbfull()->SetDBOptions({{"delayed_write_rate", "20000"}}));
ASSERT_EQ(20000, dbfull()->TEST_write_controler().max_delayed_write_rate()); ASSERT_EQ(20000, dbfull()->TEST_write_controler().max_delayed_write_rate());

@ -270,7 +270,8 @@ void GetExpectedTableProperties(
const int kDeletionCount = kTableCount * kDeletionsPerTable; const int kDeletionCount = kTableCount * kDeletionsPerTable;
const int kMergeCount = kTableCount * kMergeOperandsPerTable; const int kMergeCount = kTableCount * kMergeOperandsPerTable;
const int kRangeDeletionCount = kTableCount * kRangeDeletionsPerTable; const int kRangeDeletionCount = kTableCount * kRangeDeletionsPerTable;
const int kKeyCount = kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount; const int kKeyCount =
kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount;
const int kAvgSuccessorSize = kKeySize / 5; const int kAvgSuccessorSize = kKeySize / 5;
const int kEncodingSavePerKey = kKeySize / 4; const int kEncodingSavePerKey = kKeySize / 4;
expected_tp->raw_key_size = kKeyCount * (kKeySize + 8); expected_tp->raw_key_size = kKeyCount * (kKeySize + 8);
@ -281,7 +282,8 @@ void GetExpectedTableProperties(
expected_tp->num_merge_operands = kMergeCount; expected_tp->num_merge_operands = kMergeCount;
expected_tp->num_range_deletions = kRangeDeletionCount; expected_tp->num_range_deletions = kRangeDeletionCount;
expected_tp->num_data_blocks = expected_tp->num_data_blocks =
kTableCount * (kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) / kTableCount *
(kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) /
kBlockSize; kBlockSize;
expected_tp->data_size = expected_tp->data_size =
kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize)); kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize));
@ -1120,7 +1122,8 @@ class CountingUserTblPropCollector : public TablePropertiesCollector {
std::string encoded; std::string encoded;
PutVarint32(&encoded, count_); PutVarint32(&encoded, count_);
*properties = UserCollectedProperties{ *properties = UserCollectedProperties{
{"CountingUserTblPropCollector", message_}, {"Count", encoded}, {"CountingUserTblPropCollector", message_},
{"Count", encoded},
}; };
return Status::OK(); return Status::OK();
} }
@ -2122,7 +2125,7 @@ std::string PopMetaIndexKey(InternalIterator* meta_iter) {
} }
} }
} // namespace } // anonymous namespace
TEST_F(DBPropertiesTest, TableMetaIndexKeys) { TEST_F(DBPropertiesTest, TableMetaIndexKeys) {
// This is to detect unexpected churn in metaindex block keys. This is more // This is to detect unexpected churn in metaindex block keys. This is more

@ -238,7 +238,8 @@ TEST_F(DBRangeDelTest, SentinelsOmittedFromOutputFile) {
const Snapshot* snapshot = db_->GetSnapshot(); const Snapshot* snapshot = db_->GetSnapshot();
// gaps between ranges creates sentinels in our internal representation // gaps between ranges creates sentinels in our internal representation
std::vector<std::pair<std::string, std::string>> range_dels = {{"a", "b"}, {"c", "d"}, {"e", "f"}}; std::vector<std::pair<std::string, std::string>> range_dels = {
{"a", "b"}, {"c", "d"}, {"e", "f"}};
for (const auto& range_del : range_dels) { for (const auto& range_del : range_dels) {
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
range_del.first, range_del.second)); range_del.first, range_del.second));
@ -567,8 +568,8 @@ TEST_F(DBRangeDelTest, PutDeleteRangeMergeFlush) {
std::string val; std::string val;
PutFixed64(&val, 1); PutFixed64(&val, 1);
ASSERT_OK(db_->Put(WriteOptions(), "key", val)); ASSERT_OK(db_->Put(WriteOptions(), "key", val));
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "key",
"key", "key_")); "key_"));
ASSERT_OK(db_->Merge(WriteOptions(), "key", val)); ASSERT_OK(db_->Merge(WriteOptions(), "key", val));
ASSERT_OK(db_->Flush(FlushOptions())); ASSERT_OK(db_->Flush(FlushOptions()));
@ -1332,7 +1333,7 @@ TEST_F(DBRangeDelTest, UntruncatedTombstoneDoesNotDeleteNewerKey) {
const int kFileBytes = 1 << 20; const int kFileBytes = 1 << 20;
const int kValueBytes = 1 << 10; const int kValueBytes = 1 << 10;
const int kNumFiles = 4; const int kNumFiles = 4;
const int kMaxKey = kNumFiles* kFileBytes / kValueBytes; const int kMaxKey = kNumFiles * kFileBytes / kValueBytes;
const int kKeysOverwritten = 10; const int kKeysOverwritten = 10;
Options options = CurrentOptions(); Options options = CurrentOptions();
@ -1649,7 +1650,8 @@ TEST_F(DBRangeDelTest, RangeTombstoneWrittenToMinimalSsts) {
const auto& table_props = name_and_table_props.second; const auto& table_props = name_and_table_props.second;
// The range tombstone should only be output to the second L1 SST. // The range tombstone should only be output to the second L1 SST.
if (name.size() >= l1_metadata[1].name.size() && if (name.size() >= l1_metadata[1].name.size() &&
name.substr(name.size() - l1_metadata[1].name.size()).compare(l1_metadata[1].name) == 0) { name.substr(name.size() - l1_metadata[1].name.size())
.compare(l1_metadata[1].name) == 0) {
ASSERT_EQ(1, table_props->num_range_deletions); ASSERT_EQ(1, table_props->num_range_deletions);
++num_range_deletions; ++num_range_deletions;
} else { } else {

@ -499,7 +499,7 @@ class TraceFileEnv : public EnvWrapper {
private: private:
std::atomic<int> files_closed_{0}; std::atomic<int> files_closed_{0};
}; };
} // namespace } // anonymous namespace
TEST_F(DBSecondaryTest, SecondaryCloseFiles) { TEST_F(DBSecondaryTest, SecondaryCloseFiles) {
Options options; Options options;

@ -70,9 +70,9 @@ TEST_F(DBStatisticsTest, CompressionStatsTest) {
options.compression = kNoCompression; options.compression = kNoCompression;
DestroyAndReopen(options); DestroyAndReopen(options);
uint64_t currentCompressions = uint64_t currentCompressions =
options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED); options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
uint64_t currentDecompressions = uint64_t currentDecompressions =
options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED); options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED);
// Check that compressions do not occur when turned off // Check that compressions do not occur when turned off
for (int i = 0; i < kNumKeysWritten; ++i) { for (int i = 0; i < kNumKeysWritten; ++i) {
@ -80,14 +80,16 @@ TEST_F(DBStatisticsTest, CompressionStatsTest) {
ASSERT_OK(Put(Key(i), rnd.RandomString(128) + std::string(128, 'a'))); ASSERT_OK(Put(Key(i), rnd.RandomString(128) + std::string(128, 'a')));
} }
ASSERT_OK(Flush()); ASSERT_OK(Flush());
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED) ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED) -
- currentCompressions, 0); currentCompressions,
0);
for (int i = 0; i < kNumKeysWritten; ++i) { for (int i = 0; i < kNumKeysWritten; ++i) {
auto r = Get(Key(i)); auto r = Get(Key(i));
} }
ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED) ASSERT_EQ(options.statistics->getTickerCount(NUMBER_BLOCK_DECOMPRESSED) -
- currentDecompressions, 0); currentDecompressions,
0);
} }
TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) { TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) {

@ -52,7 +52,7 @@ void VerifyTableProperties(DB* db, uint64_t expected_entries_size) {
VerifySstUniqueIds(props); VerifySstUniqueIds(props);
} }
} // namespace } // anonymous namespace
class DBTablePropertiesTest : public DBTestBase, class DBTablePropertiesTest : public DBTestBase,
public testing::WithParamInterface<std::string> { public testing::WithParamInterface<std::string> {
@ -240,7 +240,6 @@ TablePropertiesCollection
DBTablePropertiesTest::TestGetPropertiesOfTablesInRange( DBTablePropertiesTest::TestGetPropertiesOfTablesInRange(
std::vector<Range> ranges, std::size_t* num_properties, std::vector<Range> ranges, std::size_t* num_properties,
std::size_t* num_files) { std::size_t* num_files) {
// Since we deref zero element in the vector it can not be empty // Since we deref zero element in the vector it can not be empty
// otherwise we pass an address to some random memory // otherwise we pass an address to some random memory
EXPECT_GT(ranges.size(), 0U); EXPECT_GT(ranges.size(), 0U);
@ -469,12 +468,12 @@ INSTANTIATE_TEST_CASE_P(
class DeletionTriggeredCompactionTestListener : public EventListener { class DeletionTriggeredCompactionTestListener : public EventListener {
public: public:
void OnCompactionBegin(DB* , const CompactionJobInfo& ci) override { void OnCompactionBegin(DB*, const CompactionJobInfo& ci) override {
ASSERT_EQ(ci.compaction_reason, ASSERT_EQ(ci.compaction_reason,
CompactionReason::kFilesMarkedForCompaction); CompactionReason::kFilesMarkedForCompaction);
} }
void OnCompactionCompleted(DB* , const CompactionJobInfo& ci) override { void OnCompactionCompleted(DB*, const CompactionJobInfo& ci) override {
ASSERT_EQ(ci.compaction_reason, ASSERT_EQ(ci.compaction_reason,
CompactionReason::kFilesMarkedForCompaction); CompactionReason::kFilesMarkedForCompaction);
} }
@ -485,13 +484,13 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) {
int kWindowSize = 100; int kWindowSize = 100;
int kNumDelsTrigger = 90; int kNumDelsTrigger = 90;
std::shared_ptr<TablePropertiesCollectorFactory> compact_on_del = std::shared_ptr<TablePropertiesCollectorFactory> compact_on_del =
NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger); NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger);
Options opts = CurrentOptions(); Options opts = CurrentOptions();
opts.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); opts.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
opts.table_properties_collector_factories.emplace_back(compact_on_del); opts.table_properties_collector_factories.emplace_back(compact_on_del);
if(GetParam() == "kCompactionStyleUniversal") { if (GetParam() == "kCompactionStyleUniversal") {
opts.compaction_style = kCompactionStyleUniversal; opts.compaction_style = kCompactionStyleUniversal;
} }
Reopen(opts); Reopen(opts);
@ -502,8 +501,8 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) {
ASSERT_OK(Flush()); ASSERT_OK(Flush());
MoveFilesToLevel(1); MoveFilesToLevel(1);
DeletionTriggeredCompactionTestListener *listener = DeletionTriggeredCompactionTestListener* listener =
new DeletionTriggeredCompactionTestListener(); new DeletionTriggeredCompactionTestListener();
opts.listeners.emplace_back(listener); opts.listeners.emplace_back(listener);
Reopen(opts); Reopen(opts);
@ -524,10 +523,10 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) {
// effect // effect
kWindowSize = 50; kWindowSize = 50;
kNumDelsTrigger = 40; kNumDelsTrigger = 40;
static_cast<CompactOnDeletionCollectorFactory*> static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
(compact_on_del.get())->SetWindowSize(kWindowSize); ->SetWindowSize(kWindowSize);
static_cast<CompactOnDeletionCollectorFactory*> static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
(compact_on_del.get())->SetDeletionTrigger(kNumDelsTrigger); ->SetDeletionTrigger(kNumDelsTrigger);
for (int i = 0; i < kNumKeys; ++i) { for (int i = 0; i < kNumKeys; ++i) {
if (i >= kNumKeys - kWindowSize && if (i >= kNumKeys - kWindowSize &&
i < kNumKeys - kWindowSize + kNumDelsTrigger) { i < kNumKeys - kWindowSize + kNumDelsTrigger) {
@ -543,10 +542,10 @@ TEST_P(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) {
// Change the window size to disable delete triggered compaction // Change the window size to disable delete triggered compaction
kWindowSize = 0; kWindowSize = 0;
static_cast<CompactOnDeletionCollectorFactory*> static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
(compact_on_del.get())->SetWindowSize(kWindowSize); ->SetWindowSize(kWindowSize);
static_cast<CompactOnDeletionCollectorFactory*> static_cast<CompactOnDeletionCollectorFactory*>(compact_on_del.get())
(compact_on_del.get())->SetDeletionTrigger(kNumDelsTrigger); ->SetDeletionTrigger(kNumDelsTrigger);
for (int i = 0; i < kNumKeys; ++i) { for (int i = 0; i < kNumKeys; ++i) {
if (i >= kNumKeys - kWindowSize && if (i >= kNumKeys - kWindowSize &&
i < kNumKeys - kWindowSize + kNumDelsTrigger) { i < kNumKeys - kWindowSize + kNumDelsTrigger) {
@ -611,13 +610,9 @@ TEST_P(DBTablePropertiesTest, RatioBasedDeletionTriggeredCompactionMarking) {
} }
} }
INSTANTIATE_TEST_CASE_P( INSTANTIATE_TEST_CASE_P(DBTablePropertiesTest, DBTablePropertiesTest,
DBTablePropertiesTest, ::testing::Values("kCompactionStyleLevel",
DBTablePropertiesTest, "kCompactionStyleUniversal"));
::testing::Values(
"kCompactionStyleLevel",
"kCompactionStyleUniversal"
));
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -399,7 +399,7 @@ TEST_P(DBTestTailingIterator, TailingIteratorSeekToSame) {
// Write rows with keys 00000, 00002, 00004 etc. // Write rows with keys 00000, 00002, 00004 etc.
for (int i = 0; i < NROWS; ++i) { for (int i = 0; i < NROWS; ++i) {
char buf[100]; char buf[100];
snprintf(buf, sizeof(buf), "%05d", 2*i); snprintf(buf, sizeof(buf), "%05d", 2 * i);
std::string key(buf); std::string key(buf);
std::string value("value"); std::string value("value");
ASSERT_OK(db_->Put(WriteOptions(), key, value)); ASSERT_OK(db_->Put(WriteOptions(), key, value));
@ -539,7 +539,6 @@ TEST_P(DBTestTailingIterator, SeekWithUpperBoundBug) {
const Slice upper_bound("cc", 3); const Slice upper_bound("cc", 3);
read_options.iterate_upper_bound = &upper_bound; read_options.iterate_upper_bound = &upper_bound;
// 1st L0 file // 1st L0 file
ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN")); ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN"));
ASSERT_OK(Flush()); ASSERT_OK(Flush());
@ -565,7 +564,6 @@ TEST_P(DBTestTailingIterator, SeekToFirstWithUpperBoundBug) {
const Slice upper_bound("cc", 3); const Slice upper_bound("cc", 3);
read_options.iterate_upper_bound = &upper_bound; read_options.iterate_upper_bound = &upper_bound;
// 1st L0 file // 1st L0 file
ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN")); ASSERT_OK(db_->Put(WriteOptions(), "aa", "SEEN"));
ASSERT_OK(Flush()); ASSERT_OK(Flush());
@ -599,8 +597,8 @@ int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
#else #else
(void) argc; (void)argc;
(void) argv; (void)argv;
return 0; return 0;
#endif #endif
} }

@ -1135,7 +1135,7 @@ class DelayFilterFactory : public CompactionFilterFactory {
private: private:
DBTestBase* db_test; DBTestBase* db_test;
}; };
} // namespace } // anonymous namespace
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
@ -1490,7 +1490,7 @@ bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
} }
return true; return true;
} }
} // namespace } // anonymous namespace
TEST_F(DBTest, MinLevelToCompress1) { TEST_F(DBTest, MinLevelToCompress1) {
Options options = CurrentOptions(); Options options = CurrentOptions();
@ -2843,7 +2843,7 @@ static void MTThreadBody(void* arg) {
fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter)); fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
} }
} // namespace } // anonymous namespace
class MultiThreadedDBTest class MultiThreadedDBTest
: public DBTest, : public DBTest,
@ -2929,7 +2929,7 @@ static void GCThreadBody(void* arg) {
t->done = true; t->done = true;
} }
} // namespace } // anonymous namespace
TEST_F(DBTest, GroupCommitTest) { TEST_F(DBTest, GroupCommitTest) {
do { do {
@ -4645,7 +4645,7 @@ void VerifyOperationCount(Env* env, ThreadStatus::OperationType op_type,
} }
ASSERT_EQ(op_count, expected_count); ASSERT_EQ(op_count, expected_count);
} }
} // namespace } // anonymous namespace
TEST_F(DBTest, GetThreadStatus) { TEST_F(DBTest, GetThreadStatus) {
Options options; Options options;

@ -669,33 +669,33 @@ TEST_F(DBTest2, TestWriteBufferNoLimitWithCache) {
} }
namespace { namespace {
void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist, void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
const std::vector<Slice>& keys_must_not_exist) { const std::vector<Slice>& keys_must_not_exist) {
// Ensure that expected keys exist // Ensure that expected keys exist
std::vector<std::string> values; std::vector<std::string> values;
if (keys_must_exist.size() > 0) { if (keys_must_exist.size() > 0) {
std::vector<Status> status_list = std::vector<Status> status_list =
db->MultiGet(ReadOptions(), keys_must_exist, &values); db->MultiGet(ReadOptions(), keys_must_exist, &values);
for (size_t i = 0; i < keys_must_exist.size(); i++) { for (size_t i = 0; i < keys_must_exist.size(); i++) {
ASSERT_OK(status_list[i]); ASSERT_OK(status_list[i]);
}
} }
}
// Ensure that given keys don't exist // Ensure that given keys don't exist
if (keys_must_not_exist.size() > 0) { if (keys_must_not_exist.size() > 0) {
std::vector<Status> status_list = std::vector<Status> status_list =
db->MultiGet(ReadOptions(), keys_must_not_exist, &values); db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
for (size_t i = 0; i < keys_must_not_exist.size(); i++) { for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
ASSERT_TRUE(status_list[i].IsNotFound()); ASSERT_TRUE(status_list[i].IsNotFound());
}
} }
} }
}
} // namespace } // anonymous namespace
TEST_F(DBTest2, WalFilterTest) { TEST_F(DBTest2, WalFilterTest) {
class TestWalFilter : public WalFilter { class TestWalFilter : public WalFilter {
private: private:
// Processing option that is requested to be applied at the given index // Processing option that is requested to be applied at the given index
WalFilter::WalProcessingOption wal_processing_option_; WalFilter::WalProcessingOption wal_processing_option_;
// Index at which to apply wal_processing_option_ // Index at which to apply wal_processing_option_
@ -705,12 +705,12 @@ TEST_F(DBTest2, WalFilterTest) {
// Current record index, incremented with each record encountered. // Current record index, incremented with each record encountered.
size_t current_record_index_; size_t current_record_index_;
public: public:
TestWalFilter(WalFilter::WalProcessingOption wal_processing_option, TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
size_t apply_option_for_record_index) size_t apply_option_for_record_index)
: wal_processing_option_(wal_processing_option), : wal_processing_option_(wal_processing_option),
apply_option_at_record_index_(apply_option_for_record_index), apply_option_at_record_index_(apply_option_for_record_index),
current_record_index_(0) {} current_record_index_(0) {}
WalProcessingOption LogRecord(const WriteBatch& /*batch*/, WalProcessingOption LogRecord(const WriteBatch& /*batch*/,
WriteBatch* /*new_batch*/, WriteBatch* /*new_batch*/,
@ -719,8 +719,7 @@ TEST_F(DBTest2, WalFilterTest) {
if (current_record_index_ == apply_option_at_record_index_) { if (current_record_index_ == apply_option_at_record_index_) {
option_to_return = wal_processing_option_; option_to_return = wal_processing_option_;
} } else {
else {
option_to_return = WalProcessingOption::kContinueProcessing; option_to_return = WalProcessingOption::kContinueProcessing;
} }
@ -747,12 +746,12 @@ TEST_F(DBTest2, WalFilterTest) {
// Test with all WAL processing options // Test with all WAL processing options
for (int option = 0; for (int option = 0;
option < static_cast<int>( option < static_cast<int>(
WalFilter::WalProcessingOption::kWalProcessingOptionMax); WalFilter::WalProcessingOption::kWalProcessingOptionMax);
option++) { option++) {
Options options = OptionsForLogIterTest(); Options options = OptionsForLogIterTest();
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({ "pikachu" }, options); CreateAndReopenWithCF({"pikachu"}, options);
// Write given keys in given batches // Write given keys in given batches
for (size_t i = 0; i < batch_keys.size(); i++) { for (size_t i = 0; i < batch_keys.size(); i++) {
@ -764,28 +763,27 @@ TEST_F(DBTest2, WalFilterTest) {
} }
WalFilter::WalProcessingOption wal_processing_option = WalFilter::WalProcessingOption wal_processing_option =
static_cast<WalFilter::WalProcessingOption>(option); static_cast<WalFilter::WalProcessingOption>(option);
// Create a test filter that would apply wal_processing_option at the first // Create a test filter that would apply wal_processing_option at the first
// record // record
size_t apply_option_for_record_index = 1; size_t apply_option_for_record_index = 1;
TestWalFilter test_wal_filter(wal_processing_option, TestWalFilter test_wal_filter(wal_processing_option,
apply_option_for_record_index); apply_option_for_record_index);
// Reopen database with option to use WAL filter // Reopen database with option to use WAL filter
options = OptionsForLogIterTest(); options = OptionsForLogIterTest();
options.wal_filter = &test_wal_filter; options.wal_filter = &test_wal_filter;
Status status = Status status =
TryReopenWithColumnFamilies({ "default", "pikachu" }, options); TryReopenWithColumnFamilies({"default", "pikachu"}, options);
if (wal_processing_option == if (wal_processing_option ==
WalFilter::WalProcessingOption::kCorruptedRecord) { WalFilter::WalProcessingOption::kCorruptedRecord) {
ASSERT_NOK(status); ASSERT_NOK(status);
// In case of corruption we can turn off paranoid_checks to reopen // In case of corruption we can turn off paranoid_checks to reopen
// databse // databse
options.paranoid_checks = false; options.paranoid_checks = false;
ReopenWithColumnFamilies({ "default", "pikachu" }, options); ReopenWithColumnFamilies({"default", "pikachu"}, options);
} } else {
else {
ASSERT_OK(status); ASSERT_OK(status);
} }
@ -794,56 +792,54 @@ TEST_F(DBTest2, WalFilterTest) {
std::vector<Slice> keys_must_exist; std::vector<Slice> keys_must_exist;
std::vector<Slice> keys_must_not_exist; std::vector<Slice> keys_must_not_exist;
switch (wal_processing_option) { switch (wal_processing_option) {
case WalFilter::WalProcessingOption::kCorruptedRecord: case WalFilter::WalProcessingOption::kCorruptedRecord:
case WalFilter::WalProcessingOption::kContinueProcessing: { case WalFilter::WalProcessingOption::kContinueProcessing: {
fprintf(stderr, "Testing with complete WAL processing\n"); fprintf(stderr, "Testing with complete WAL processing\n");
// we expect all records to be processed // we expect all records to be processed
for (size_t i = 0; i < batch_keys.size(); i++) { for (size_t i = 0; i < batch_keys.size(); i++) {
for (size_t j = 0; j < batch_keys[i].size(); j++) { for (size_t j = 0; j < batch_keys[i].size(); j++) {
keys_must_exist.push_back(Slice(batch_keys[i][j]));
}
}
break;
}
case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
fprintf(stderr,
"Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
apply_option_for_record_index);
// We expect the record with apply_option_for_record_index to be not
// found.
for (size_t i = 0; i < batch_keys.size(); i++) {
for (size_t j = 0; j < batch_keys[i].size(); j++) {
if (i == apply_option_for_record_index) {
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
}
else {
keys_must_exist.push_back(Slice(batch_keys[i][j])); keys_must_exist.push_back(Slice(batch_keys[i][j]));
} }
} }
break;
} }
break; case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
} fprintf(stderr,
case WalFilter::WalProcessingOption::kStopReplay: { "Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
fprintf(stderr, apply_option_for_record_index);
"Testing with stopping replay from record %" ROCKSDB_PRIszt // We expect the record with apply_option_for_record_index to be not
"\n", // found.
apply_option_for_record_index); for (size_t i = 0; i < batch_keys.size(); i++) {
// We expect records beyond apply_option_for_record_index to be not for (size_t j = 0; j < batch_keys[i].size(); j++) {
// found. if (i == apply_option_for_record_index) {
for (size_t i = 0; i < batch_keys.size(); i++) { keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
for (size_t j = 0; j < batch_keys[i].size(); j++) { } else {
if (i >= apply_option_for_record_index) { keys_must_exist.push_back(Slice(batch_keys[i][j]));
keys_must_not_exist.push_back(Slice(batch_keys[i][j])); }
} }
else { }
keys_must_exist.push_back(Slice(batch_keys[i][j])); break;
}
case WalFilter::WalProcessingOption::kStopReplay: {
fprintf(stderr,
"Testing with stopping replay from record %" ROCKSDB_PRIszt
"\n",
apply_option_for_record_index);
// We expect records beyond apply_option_for_record_index to be not
// found.
for (size_t i = 0; i < batch_keys.size(); i++) {
for (size_t j = 0; j < batch_keys[i].size(); j++) {
if (i >= apply_option_for_record_index) {
keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
} else {
keys_must_exist.push_back(Slice(batch_keys[i][j]));
}
} }
} }
break;
} }
break; default:
} FAIL(); // unhandled case
default:
FAIL(); // unhandled case
} }
bool checked_after_reopen = false; bool checked_after_reopen = false;
@ -861,7 +857,7 @@ TEST_F(DBTest2, WalFilterTest) {
//(even if they were skipped) //(even if they were skipped)
// reopn database with option to use WAL filter // reopn database with option to use WAL filter
options = OptionsForLogIterTest(); options = OptionsForLogIterTest();
ReopenWithColumnFamilies({ "default", "pikachu" }, options); ReopenWithColumnFamilies({"default", "pikachu"}, options);
checked_after_reopen = true; checked_after_reopen = true;
} }
@ -870,7 +866,7 @@ TEST_F(DBTest2, WalFilterTest) {
TEST_F(DBTest2, WalFilterTestWithChangeBatch) { TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
class ChangeBatchHandler : public WriteBatch::Handler { class ChangeBatchHandler : public WriteBatch::Handler {
private: private:
// Batch to insert keys in // Batch to insert keys in
WriteBatch* new_write_batch_; WriteBatch* new_write_batch_;
// Number of keys to add in the new batch // Number of keys to add in the new batch
@ -878,12 +874,12 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
// Number of keys added to new batch // Number of keys added to new batch
size_t num_keys_added_; size_t num_keys_added_;
public: public:
ChangeBatchHandler(WriteBatch* new_write_batch, ChangeBatchHandler(WriteBatch* new_write_batch,
size_t num_keys_to_add_in_new_batch) size_t num_keys_to_add_in_new_batch)
: new_write_batch_(new_write_batch), : new_write_batch_(new_write_batch),
num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch), num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
num_keys_added_(0) {} num_keys_added_(0) {}
void Put(const Slice& key, const Slice& value) override { void Put(const Slice& key, const Slice& value) override {
if (num_keys_added_ < num_keys_to_add_in_new_batch_) { if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
ASSERT_OK(new_write_batch_->Put(key, value)); ASSERT_OK(new_write_batch_->Put(key, value));
@ -893,7 +889,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
}; };
class TestWalFilterWithChangeBatch : public WalFilter { class TestWalFilterWithChangeBatch : public WalFilter {
private: private:
// Index at which to start changing records // Index at which to start changing records
size_t change_records_from_index_; size_t change_records_from_index_;
// Number of keys to add in the new batch // Number of keys to add in the new batch
@ -901,12 +897,12 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
// Current record index, incremented with each record encountered. // Current record index, incremented with each record encountered.
size_t current_record_index_; size_t current_record_index_;
public: public:
TestWalFilterWithChangeBatch(size_t change_records_from_index, TestWalFilterWithChangeBatch(size_t change_records_from_index,
size_t num_keys_to_add_in_new_batch) size_t num_keys_to_add_in_new_batch)
: change_records_from_index_(change_records_from_index), : change_records_from_index_(change_records_from_index),
num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch), num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
current_record_index_(0) {} current_record_index_(0) {}
WalProcessingOption LogRecord(const WriteBatch& batch, WalProcessingOption LogRecord(const WriteBatch& batch,
WriteBatch* new_batch, WriteBatch* new_batch,
@ -925,7 +921,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
// object, however we modify it for our own purpose here and hence // object, however we modify it for our own purpose here and hence
// cast the constness away. // cast the constness away.
(const_cast<TestWalFilterWithChangeBatch*>(this) (const_cast<TestWalFilterWithChangeBatch*>(this)
->current_record_index_)++; ->current_record_index_)++;
return WalProcessingOption::kContinueProcessing; return WalProcessingOption::kContinueProcessing;
} }
@ -944,7 +940,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
Options options = OptionsForLogIterTest(); Options options = OptionsForLogIterTest();
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({ "pikachu" }, options); CreateAndReopenWithCF({"pikachu"}, options);
// Write given keys in given batches // Write given keys in given batches
for (size_t i = 0; i < batch_keys.size(); i++) { for (size_t i = 0; i < batch_keys.size(); i++) {
@ -960,12 +956,12 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
size_t change_records_from_index = 1; size_t change_records_from_index = 1;
size_t num_keys_to_add_in_new_batch = 1; size_t num_keys_to_add_in_new_batch = 1;
TestWalFilterWithChangeBatch test_wal_filter_with_change_batch( TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
change_records_from_index, num_keys_to_add_in_new_batch); change_records_from_index, num_keys_to_add_in_new_batch);
// Reopen database with option to use WAL filter // Reopen database with option to use WAL filter
options = OptionsForLogIterTest(); options = OptionsForLogIterTest();
options.wal_filter = &test_wal_filter_with_change_batch; options.wal_filter = &test_wal_filter_with_change_batch;
ReopenWithColumnFamilies({ "default", "pikachu" }, options); ReopenWithColumnFamilies({"default", "pikachu"}, options);
// Ensure that all keys exist before change_records_from_index_ // Ensure that all keys exist before change_records_from_index_
// And after that index only single key exists // And after that index only single key exists
@ -977,8 +973,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
for (size_t j = 0; j < batch_keys[i].size(); j++) { for (size_t j = 0; j < batch_keys[i].size(); j++) {
if (i >= change_records_from_index && j >= num_keys_to_add_in_new_batch) { if (i >= change_records_from_index && j >= num_keys_to_add_in_new_batch) {
keys_must_not_exist.push_back(Slice(batch_keys[i][j])); keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
} } else {
else {
keys_must_exist.push_back(Slice(batch_keys[i][j])); keys_must_exist.push_back(Slice(batch_keys[i][j]));
} }
} }
@ -999,7 +994,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
//(even if they were skipped) //(even if they were skipped)
// reopn database with option to use WAL filter // reopn database with option to use WAL filter
options = OptionsForLogIterTest(); options = OptionsForLogIterTest();
ReopenWithColumnFamilies({ "default", "pikachu" }, options); ReopenWithColumnFamilies({"default", "pikachu"}, options);
checked_after_reopen = true; checked_after_reopen = true;
} }
@ -1007,22 +1002,23 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) { TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter { class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
public: public:
WalProcessingOption LogRecord(const WriteBatch& batch, WriteBatch* new_batch, WalProcessingOption LogRecord(const WriteBatch& batch,
bool* batch_changed) const override { WriteBatch* new_batch,
*new_batch = batch; bool* batch_changed) const override {
Status s = new_batch->Put("key_extra", "value_extra"); *new_batch = batch;
if (s.ok()) { Status s = new_batch->Put("key_extra", "value_extra");
*batch_changed = true; if (s.ok()) {
} else { *batch_changed = true;
assert(false); } else {
} assert(false);
return WalProcessingOption::kContinueProcessing; }
} return WalProcessingOption::kContinueProcessing;
}
const char* Name() const override {
return "WalFilterTestWithChangeBatchExtraKeys"; const char* Name() const override {
} return "WalFilterTestWithChangeBatchExtraKeys";
}
}; };
std::vector<std::vector<std::string>> batch_keys(3); std::vector<std::vector<std::string>> batch_keys(3);
@ -1036,7 +1032,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
Options options = OptionsForLogIterTest(); Options options = OptionsForLogIterTest();
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({ "pikachu" }, options); CreateAndReopenWithCF({"pikachu"}, options);
// Write given keys in given batches // Write given keys in given batches
for (size_t i = 0; i < batch_keys.size(); i++) { for (size_t i = 0; i < batch_keys.size(); i++) {
@ -1059,7 +1055,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
// Reopen without filter, now reopen should succeed - previous // Reopen without filter, now reopen should succeed - previous
// attempt to open must not have altered the db. // attempt to open must not have altered the db.
options = OptionsForLogIterTest(); options = OptionsForLogIterTest();
ReopenWithColumnFamilies({ "default", "pikachu" }, options); ReopenWithColumnFamilies({"default", "pikachu"}, options);
std::vector<Slice> keys_must_exist; std::vector<Slice> keys_must_exist;
std::vector<Slice> keys_must_not_exist; // empty vector std::vector<Slice> keys_must_not_exist; // empty vector
@ -1075,7 +1071,7 @@ TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
TEST_F(DBTest2, WalFilterTestWithColumnFamilies) { TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
class TestWalFilterWithColumnFamilies : public WalFilter { class TestWalFilterWithColumnFamilies : public WalFilter {
private: private:
// column_family_id -> log_number map (provided to WALFilter) // column_family_id -> log_number map (provided to WALFilter)
std::map<uint32_t, uint64_t> cf_log_number_map_; std::map<uint32_t, uint64_t> cf_log_number_map_;
// column_family_name -> column_family_id map (provided to WALFilter) // column_family_name -> column_family_id map (provided to WALFilter)
@ -1085,31 +1081,34 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
// during recovery (i.e. aren't already flushed to SST file(s)) // during recovery (i.e. aren't already flushed to SST file(s))
// for verification against the keys we expect. // for verification against the keys we expect.
std::map<uint32_t, std::vector<std::string>> cf_wal_keys_; std::map<uint32_t, std::vector<std::string>> cf_wal_keys_;
public:
void ColumnFamilyLogNumberMap( public:
const std::map<uint32_t, uint64_t>& cf_lognumber_map, void ColumnFamilyLogNumberMap(
const std::map<std::string, uint32_t>& cf_name_id_map) override { const std::map<uint32_t, uint64_t>& cf_lognumber_map,
cf_log_number_map_ = cf_lognumber_map; const std::map<std::string, uint32_t>& cf_name_id_map) override {
cf_name_id_map_ = cf_name_id_map; cf_log_number_map_ = cf_lognumber_map;
} cf_name_id_map_ = cf_name_id_map;
}
WalProcessingOption LogRecordFound(unsigned long long log_number,
const std::string& /*log_file_name*/, WalProcessingOption LogRecordFound(unsigned long long log_number,
const WriteBatch& batch, const std::string& /*log_file_name*/,
WriteBatch* /*new_batch*/, const WriteBatch& batch,
bool* /*batch_changed*/) override { WriteBatch* /*new_batch*/,
class LogRecordBatchHandler : public WriteBatch::Handler { bool* /*batch_changed*/) override {
private: class LogRecordBatchHandler : public WriteBatch::Handler {
const std::map<uint32_t, uint64_t> & cf_log_number_map_; private:
std::map<uint32_t, std::vector<std::string>> & cf_wal_keys_; const std::map<uint32_t, uint64_t>& cf_log_number_map_;
std::map<uint32_t, std::vector<std::string>>& cf_wal_keys_;
unsigned long long log_number_; unsigned long long log_number_;
public:
LogRecordBatchHandler(unsigned long long current_log_number, public:
const std::map<uint32_t, uint64_t> & cf_log_number_map, LogRecordBatchHandler(
std::map<uint32_t, std::vector<std::string>> & cf_wal_keys) : unsigned long long current_log_number,
cf_log_number_map_(cf_log_number_map), const std::map<uint32_t, uint64_t>& cf_log_number_map,
cf_wal_keys_(cf_wal_keys), std::map<uint32_t, std::vector<std::string>>& cf_wal_keys)
log_number_(current_log_number){} : cf_log_number_map_(cf_log_number_map),
cf_wal_keys_(cf_wal_keys),
log_number_(current_log_number) {}
Status PutCF(uint32_t column_family_id, const Slice& key, Status PutCF(uint32_t column_family_id, const Slice& key,
const Slice& /*value*/) override { const Slice& /*value*/) override {
@ -1120,8 +1119,8 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
// (i.e. isn't flushed to SST file(s) for column_family_id) // (i.e. isn't flushed to SST file(s) for column_family_id)
// add it to the cf_wal_keys_ map for verification. // add it to the cf_wal_keys_ map for verification.
if (log_number_ >= log_number_for_cf) { if (log_number_ >= log_number_for_cf) {
cf_wal_keys_[column_family_id].push_back(std::string(key.data(), cf_wal_keys_[column_family_id].push_back(
key.size())); std::string(key.data(), key.size()));
} }
return Status::OK(); return Status::OK();
} }
@ -1134,17 +1133,17 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
} }
return WalProcessingOption::kContinueProcessing; return WalProcessingOption::kContinueProcessing;
} }
const char* Name() const override { const char* Name() const override {
return "WalFilterTestWithColumnFamilies"; return "WalFilterTestWithColumnFamilies";
} }
const std::map<uint32_t, std::vector<std::string>>& GetColumnFamilyKeys() { const std::map<uint32_t, std::vector<std::string>>& GetColumnFamilyKeys() {
return cf_wal_keys_; return cf_wal_keys_;
} }
const std::map<std::string, uint32_t> & GetColumnFamilyNameIdMap() { const std::map<std::string, uint32_t>& GetColumnFamilyNameIdMap() {
return cf_name_id_map_; return cf_name_id_map_;
} }
}; };
@ -1160,7 +1159,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
Options options = OptionsForLogIterTest(); Options options = OptionsForLogIterTest();
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({ "pikachu" }, options); CreateAndReopenWithCF({"pikachu"}, options);
// Write given keys in given batches // Write given keys in given batches
for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) { for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
@ -1174,7 +1173,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
ASSERT_OK(dbfull()->Write(WriteOptions(), &batch)); ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
} }
//Flush default column-family // Flush default column-family
ASSERT_OK(db_->Flush(FlushOptions(), handles_[0])); ASSERT_OK(db_->Flush(FlushOptions(), handles_[0]));
// Do some more writes // Do some more writes
@ -1208,8 +1207,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
// Reopen database with option to use WAL filter // Reopen database with option to use WAL filter
options = OptionsForLogIterTest(); options = OptionsForLogIterTest();
options.wal_filter = &test_wal_filter_column_families; options.wal_filter = &test_wal_filter_column_families;
Status status = Status status = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
// verify that handles_[0] only has post_flush keys // verify that handles_[0] only has post_flush keys
@ -1218,7 +1216,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
auto name_id_map = test_wal_filter_column_families.GetColumnFamilyNameIdMap(); auto name_id_map = test_wal_filter_column_families.GetColumnFamilyNameIdMap();
size_t index = 0; size_t index = 0;
auto keys_cf = cf_wal_keys[name_id_map[kDefaultColumnFamilyName]]; auto keys_cf = cf_wal_keys[name_id_map[kDefaultColumnFamilyName]];
//default column-family, only post_flush keys are expected // default column-family, only post_flush keys are expected
for (size_t i = 0; i < batch_keys_post_flush.size(); i++) { for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) { for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
Slice key_from_the_log(keys_cf[index++]); Slice key_from_the_log(keys_cf[index++]);
@ -1230,7 +1228,7 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
index = 0; index = 0;
keys_cf = cf_wal_keys[name_id_map["pikachu"]]; keys_cf = cf_wal_keys[name_id_map["pikachu"]];
//pikachu column-family, all keys are expected // pikachu column-family, all keys are expected
for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) { for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) { for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
Slice key_from_the_log(keys_cf[index++]); Slice key_from_the_log(keys_cf[index++]);
@ -1280,7 +1278,7 @@ TEST_F(DBTest2, PresetCompressionDict) {
#if LZ4_VERSION_NUMBER >= 10400 // r124+ #if LZ4_VERSION_NUMBER >= 10400 // r124+
compression_types.push_back(kLZ4Compression); compression_types.push_back(kLZ4Compression);
compression_types.push_back(kLZ4HCCompression); compression_types.push_back(kLZ4HCCompression);
#endif // LZ4_VERSION_NUMBER >= 10400 #endif // LZ4_VERSION_NUMBER >= 10400
if (ZSTD_Supported()) { if (ZSTD_Supported()) {
compression_types.push_back(kZSTD); compression_types.push_back(kZSTD);
} }
@ -1960,7 +1958,8 @@ TEST_F(DBTest2, CompressionOptions) {
class CompactionStallTestListener : public EventListener { class CompactionStallTestListener : public EventListener {
public: public:
CompactionStallTestListener() : compacting_files_cnt_(0), compacted_files_cnt_(0) {} CompactionStallTestListener()
: compacting_files_cnt_(0), compacted_files_cnt_(0) {}
void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override { void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override {
ASSERT_EQ(ci.cf_name, "default"); ASSERT_EQ(ci.cf_name, "default");
@ -2039,7 +2038,8 @@ TEST_F(DBTest2, CompactionStall) {
options.level0_file_num_compaction_trigger); options.level0_file_num_compaction_trigger);
ASSERT_GT(listener->compacted_files_cnt_.load(), ASSERT_GT(listener->compacted_files_cnt_.load(),
10 - options.level0_file_num_compaction_trigger); 10 - options.level0_file_num_compaction_trigger);
ASSERT_EQ(listener->compacting_files_cnt_.load(), listener->compacted_files_cnt_.load()); ASSERT_EQ(listener->compacting_files_cnt_.load(),
listener->compacted_files_cnt_.load());
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
} }
@ -2664,7 +2664,7 @@ namespace {
void CountSyncPoint() { void CountSyncPoint() {
TEST_SYNC_POINT_CALLBACK("DBTest2::MarkedPoint", nullptr /* arg */); TEST_SYNC_POINT_CALLBACK("DBTest2::MarkedPoint", nullptr /* arg */);
} }
} // namespace } // anonymous namespace
TEST_F(DBTest2, SyncPointMarker) { TEST_F(DBTest2, SyncPointMarker) {
std::atomic<int> sync_point_called(0); std::atomic<int> sync_point_called(0);
@ -2797,7 +2797,7 @@ TEST_F(DBTest2, ReadAmpBitmap) {
} }
} }
#ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented #ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented
TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) { TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
{ {
const int kIdBufLen = 100; const int kIdBufLen = 100;
@ -2899,7 +2899,6 @@ TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
size_t total_loaded_bytes_iter2 = size_t total_loaded_bytes_iter2 =
options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES); options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES);
// Read amp is on average 100% since we read all what we loaded in memory // Read amp is on average 100% since we read all what we loaded in memory
if (k == 0) { if (k == 0) {
ASSERT_EQ(total_useful_bytes_iter1 + total_useful_bytes_iter2, ASSERT_EQ(total_useful_bytes_iter1 + total_useful_bytes_iter2,
@ -2911,7 +2910,7 @@ TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
} }
} }
} }
#endif // !OS_SOLARIS #endif // !OS_SOLARIS
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) { TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
@ -5192,7 +5191,7 @@ TEST_F(DBTest2, TraceWithFilter) {
ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions())); ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions()));
handles.clear(); handles.clear();
DB* db3 = nullptr; DB* db3 = nullptr;
ASSERT_OK(DB::Open(db_opts, dbname3, column_families, &handles, &db3)); ASSERT_OK(DB::Open(db_opts, dbname3, column_families, &handles, &db3));
env_->SleepForMicroseconds(100); env_->SleepForMicroseconds(100);
@ -5200,12 +5199,12 @@ TEST_F(DBTest2, TraceWithFilter) {
ASSERT_TRUE(db3->Get(ro, handles[0], "a", &value).IsNotFound()); ASSERT_TRUE(db3->Get(ro, handles[0], "a", &value).IsNotFound());
ASSERT_TRUE(db3->Get(ro, handles[0], "g", &value).IsNotFound()); ASSERT_TRUE(db3->Get(ro, handles[0], "g", &value).IsNotFound());
//The tracer will not record the READ ops. // The tracer will not record the READ ops.
trace_opts.filter = TraceFilterType::kTraceFilterGet; trace_opts.filter = TraceFilterType::kTraceFilterGet;
std::string trace_filename3 = dbname_ + "/rocksdb.trace_3"; std::string trace_filename3 = dbname_ + "/rocksdb.trace_3";
std::unique_ptr<TraceWriter> trace_writer3; std::unique_ptr<TraceWriter> trace_writer3;
ASSERT_OK( ASSERT_OK(
NewFileTraceWriter(env_, env_opts, trace_filename3, &trace_writer3)); NewFileTraceWriter(env_, env_opts, trace_filename3, &trace_writer3));
ASSERT_OK(db3->StartTrace(trace_opts, std::move(trace_writer3))); ASSERT_OK(db3->StartTrace(trace_opts, std::move(trace_writer3)));
ASSERT_OK(db3->Put(wo, handles[0], "a", "1")); ASSERT_OK(db3->Put(wo, handles[0], "a", "1"));
@ -5227,7 +5226,7 @@ TEST_F(DBTest2, TraceWithFilter) {
std::unique_ptr<TraceReader> trace_reader3; std::unique_ptr<TraceReader> trace_reader3;
ASSERT_OK( ASSERT_OK(
NewFileTraceReader(env_, env_opts, trace_filename3, &trace_reader3)); NewFileTraceReader(env_, env_opts, trace_filename3, &trace_reader3));
// Count the number of records in the trace file; // Count the number of records in the trace file;
int count = 0; int count = 0;
@ -5503,16 +5502,20 @@ TEST_F(DBTest2, TestGetColumnFamilyHandleUnlocked) {
port::Thread user_thread1([&]() { port::Thread user_thread1([&]() {
auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[0]->GetID()); auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[0]->GetID());
ASSERT_EQ(cfh->GetID(), handles_[0]->GetID()); ASSERT_EQ(cfh->GetID(), handles_[0]->GetID());
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked1"); TEST_SYNC_POINT(
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::ReadColumnFamilyHandle1"); "TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked1");
TEST_SYNC_POINT(
"TestGetColumnFamilyHandleUnlocked::ReadColumnFamilyHandle1");
ASSERT_EQ(cfh->GetID(), handles_[0]->GetID()); ASSERT_EQ(cfh->GetID(), handles_[0]->GetID());
}); });
port::Thread user_thread2([&]() { port::Thread user_thread2([&]() {
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::PreGetColumnFamilyHandleUnlocked2"); TEST_SYNC_POINT(
"TestGetColumnFamilyHandleUnlocked::PreGetColumnFamilyHandleUnlocked2");
auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[1]->GetID()); auto cfh = dbi->GetColumnFamilyHandleUnlocked(handles_[1]->GetID());
ASSERT_EQ(cfh->GetID(), handles_[1]->GetID()); ASSERT_EQ(cfh->GetID(), handles_[1]->GetID());
TEST_SYNC_POINT("TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked2"); TEST_SYNC_POINT(
"TestGetColumnFamilyHandleUnlocked::GetColumnFamilyHandleUnlocked2");
ASSERT_EQ(cfh->GetID(), handles_[1]->GetID()); ASSERT_EQ(cfh->GetID(), handles_[1]->GetID());
}); });
@ -5666,7 +5669,7 @@ class DummyOldStats : public Statistics {
std::atomic<int> num_rt{0}; std::atomic<int> num_rt{0};
std::atomic<int> num_mt{0}; std::atomic<int> num_mt{0};
}; };
} // namespace } // anonymous namespace
TEST_F(DBTest2, OldStatsInterface) { TEST_F(DBTest2, OldStatsInterface) {
DummyOldStats* dos = new DummyOldStats(); DummyOldStats* dos = new DummyOldStats();

@ -29,7 +29,7 @@ int64_t MaybeCurrentTime(Env* env) {
env->GetCurrentTime(&time).PermitUncheckedError(); env->GetCurrentTime(&time).PermitUncheckedError();
return time; return time;
} }
} // namespace } // anonymous namespace
// Special Env used to delay background operations // Special Env used to delay background operations
@ -125,54 +125,54 @@ DBTestBase::~DBTestBase() {
bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) { bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) {
#ifdef ROCKSDB_LITE #ifdef ROCKSDB_LITE
// These options are not supported in ROCKSDB_LITE // These options are not supported in ROCKSDB_LITE
if (option_config == kHashSkipList || if (option_config == kHashSkipList ||
option_config == kPlainTableFirstBytePrefix || option_config == kPlainTableFirstBytePrefix ||
option_config == kPlainTableCappedPrefix || option_config == kPlainTableCappedPrefix ||
option_config == kPlainTableCappedPrefixNonMmap || option_config == kPlainTableCappedPrefixNonMmap ||
option_config == kPlainTableAllBytesPrefix || option_config == kPlainTableAllBytesPrefix ||
option_config == kVectorRep || option_config == kHashLinkList || option_config == kVectorRep || option_config == kHashLinkList ||
option_config == kUniversalCompaction || option_config == kUniversalCompaction ||
option_config == kUniversalCompactionMultiLevel || option_config == kUniversalCompactionMultiLevel ||
option_config == kUniversalSubcompactions || option_config == kUniversalSubcompactions ||
option_config == kFIFOCompaction || option_config == kFIFOCompaction ||
option_config == kConcurrentSkipList) { option_config == kConcurrentSkipList) {
return true; return true;
} }
#endif #endif
if ((skip_mask & kSkipUniversalCompaction) && if ((skip_mask & kSkipUniversalCompaction) &&
(option_config == kUniversalCompaction || (option_config == kUniversalCompaction ||
option_config == kUniversalCompactionMultiLevel || option_config == kUniversalCompactionMultiLevel ||
option_config == kUniversalSubcompactions)) { option_config == kUniversalSubcompactions)) {
return true; return true;
} }
if ((skip_mask & kSkipMergePut) && option_config == kMergePut) { if ((skip_mask & kSkipMergePut) && option_config == kMergePut) {
return true; return true;
} }
if ((skip_mask & kSkipNoSeekToLast) && if ((skip_mask & kSkipNoSeekToLast) &&
(option_config == kHashLinkList || option_config == kHashSkipList)) { (option_config == kHashLinkList || option_config == kHashSkipList)) {
return true; return true;
} }
if ((skip_mask & kSkipPlainTable) && if ((skip_mask & kSkipPlainTable) &&
(option_config == kPlainTableAllBytesPrefix || (option_config == kPlainTableAllBytesPrefix ||
option_config == kPlainTableFirstBytePrefix || option_config == kPlainTableFirstBytePrefix ||
option_config == kPlainTableCappedPrefix || option_config == kPlainTableCappedPrefix ||
option_config == kPlainTableCappedPrefixNonMmap)) { option_config == kPlainTableCappedPrefixNonMmap)) {
return true; return true;
} }
if ((skip_mask & kSkipHashIndex) && if ((skip_mask & kSkipHashIndex) &&
(option_config == kBlockBasedTableWithPrefixHashIndex || (option_config == kBlockBasedTableWithPrefixHashIndex ||
option_config == kBlockBasedTableWithWholeKeyHashIndex)) { option_config == kBlockBasedTableWithWholeKeyHashIndex)) {
return true; return true;
} }
if ((skip_mask & kSkipFIFOCompaction) && option_config == kFIFOCompaction) { if ((skip_mask & kSkipFIFOCompaction) && option_config == kFIFOCompaction) {
return true; return true;
} }
if ((skip_mask & kSkipMmapReads) && option_config == kWalDirAndMmapReads) { if ((skip_mask & kSkipMmapReads) && option_config == kWalDirAndMmapReads) {
return true; return true;
} }
return false; return false;
} }
// Switch to a fresh database with the next option configuration to // Switch to a fresh database with the next option configuration to
@ -424,13 +424,13 @@ Options DBTestBase::GetOptions(
options.allow_concurrent_memtable_write = false; options.allow_concurrent_memtable_write = false;
options.unordered_write = false; options.unordered_write = false;
break; break;
case kDirectIO: { case kDirectIO: {
options.use_direct_reads = true; options.use_direct_reads = true;
options.use_direct_io_for_flush_and_compaction = true; options.use_direct_io_for_flush_and_compaction = true;
options.compaction_readahead_size = 2 * 1024 * 1024; options.compaction_readahead_size = 2 * 1024 * 1024;
SetupSyncPointsToMockDirectIO(); SetupSyncPointsToMockDirectIO();
break; break;
} }
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
case kMergePut: case kMergePut:
options.merge_operator = MergeOperators::CreatePutOperator(); options.merge_operator = MergeOperators::CreatePutOperator();
@ -1308,12 +1308,14 @@ void DBTestBase::GetSstFiles(Env* env, std::string path,
std::vector<std::string>* files) { std::vector<std::string>* files) {
EXPECT_OK(env->GetChildren(path, files)); EXPECT_OK(env->GetChildren(path, files));
files->erase( files->erase(std::remove_if(files->begin(), files->end(),
std::remove_if(files->begin(), files->end(), [](std::string name) { [](std::string name) {
uint64_t number; uint64_t number;
FileType type; FileType type;
return !(ParseFileName(name, &number, &type) && type == kTableFile); return !(ParseFileName(name, &number, &type) &&
}), files->end()); type == kTableFile);
}),
files->end());
} }
int DBTestBase::GetSstFileCount(std::string path) { int DBTestBase::GetSstFileCount(std::string path) {
@ -1583,8 +1585,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
iter_cnt++; iter_cnt++;
total_reads++; total_reads++;
} }
ASSERT_EQ(data_iter, true_data.end()) << iter_cnt << " / " ASSERT_EQ(data_iter, true_data.end())
<< true_data.size(); << iter_cnt << " / " << true_data.size();
delete iter; delete iter;
// Verify Iterator::Prev() // Verify Iterator::Prev()
@ -1606,8 +1608,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
iter_cnt++; iter_cnt++;
total_reads++; total_reads++;
} }
ASSERT_EQ(data_rev, true_data.rend()) << iter_cnt << " / " ASSERT_EQ(data_rev, true_data.rend())
<< true_data.size(); << iter_cnt << " / " << true_data.size();
// Verify Iterator::Seek() // Verify Iterator::Seek()
for (auto kv : true_data) { for (auto kv : true_data) {
@ -1637,8 +1639,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
iter_cnt++; iter_cnt++;
total_reads++; total_reads++;
} }
ASSERT_EQ(data_iter, true_data.end()) << iter_cnt << " / " ASSERT_EQ(data_iter, true_data.end())
<< true_data.size(); << iter_cnt << " / " << true_data.size();
// Verify ForwardIterator::Seek() // Verify ForwardIterator::Seek()
for (auto kv : true_data) { for (auto kv : true_data) {

@ -220,9 +220,7 @@ class SpecialEnv : public EnvWrapper {
Env::IOPriority GetIOPriority() override { Env::IOPriority GetIOPriority() override {
return base_->GetIOPriority(); return base_->GetIOPriority();
} }
bool use_direct_io() const override { bool use_direct_io() const override { return base_->use_direct_io(); }
return base_->use_direct_io();
}
Status Allocate(uint64_t offset, uint64_t len) override { Status Allocate(uint64_t offset, uint64_t len) override {
return base_->Allocate(offset, len); return base_->Allocate(offset, len);
} }

@ -39,8 +39,8 @@ class DBTestUniversalCompactionBase
class DBTestUniversalCompaction : public DBTestUniversalCompactionBase { class DBTestUniversalCompaction : public DBTestUniversalCompactionBase {
public: public:
DBTestUniversalCompaction() : DBTestUniversalCompaction()
DBTestUniversalCompactionBase("/db_universal_compaction_test") {} : DBTestUniversalCompactionBase("/db_universal_compaction_test") {}
}; };
class DBTestUniversalCompaction2 : public DBTestBase { class DBTestUniversalCompaction2 : public DBTestBase {
@ -93,7 +93,7 @@ class KeepFilterFactory : public CompactionFilterFactory {
std::atomic_bool expect_full_compaction_; std::atomic_bool expect_full_compaction_;
std::atomic_bool expect_manual_compaction_; std::atomic_bool expect_manual_compaction_;
}; };
} // namespace } // anonymous namespace
// Make sure we don't trigger a problem if the trigger condtion is given // Make sure we don't trigger a problem if the trigger condtion is given
// to be 0, which is invalid. // to be 0, which is invalid.
@ -563,8 +563,7 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
} }
if (compaction_input_file_names.size() == 0) { if (compaction_input_file_names.size() == 0) {
compaction_input_file_names.push_back( compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name);
cf_meta.levels[0].files[0].name);
} }
// expect fail since universal compaction only allow L0 output // expect fail since universal compaction only allow L0 output
@ -574,28 +573,23 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
.ok()); .ok());
// expect ok and verify the compacted files no longer exist. // expect ok and verify the compacted files no longer exist.
ASSERT_OK(dbfull()->CompactFiles( ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
CompactionOptions(), handles_[1], compaction_input_file_names, 0));
compaction_input_file_names, 0));
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta); dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
VerifyCompactionResult( VerifyCompactionResult(
cf_meta, cf_meta, std::set<std::string>(compaction_input_file_names.begin(),
std::set<std::string>(compaction_input_file_names.begin(), compaction_input_file_names.end()));
compaction_input_file_names.end()));
compaction_input_file_names.clear(); compaction_input_file_names.clear();
// Pick the first and the last file, expect everything is // Pick the first and the last file, expect everything is
// compacted into one single file. // compacted into one single file.
compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name);
compaction_input_file_names.push_back( compaction_input_file_names.push_back(
cf_meta.levels[0].files[0].name); cf_meta.levels[0].files[cf_meta.levels[0].files.size() - 1].name);
compaction_input_file_names.push_back( ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
cf_meta.levels[0].files[ compaction_input_file_names, 0));
cf_meta.levels[0].files.size() - 1].name);
ASSERT_OK(dbfull()->CompactFiles(
CompactionOptions(), handles_[1],
compaction_input_file_names, 0));
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta); dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
ASSERT_EQ(cf_meta.levels[0].files.size(), 1U); ASSERT_EQ(cf_meta.levels[0].files.size(), 1U);
@ -604,7 +598,7 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) { TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.write_buffer_size = 100 << 10; // 100KB options.write_buffer_size = 100 << 10; // 100KB
options.num_levels = 7; options.num_levels = 7;
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
DestroyAndReopen(options); DestroyAndReopen(options);
@ -640,9 +634,9 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
class DBTestUniversalCompactionMultiLevels class DBTestUniversalCompactionMultiLevels
: public DBTestUniversalCompactionBase { : public DBTestUniversalCompactionBase {
public: public:
DBTestUniversalCompactionMultiLevels() : DBTestUniversalCompactionMultiLevels()
DBTestUniversalCompactionBase( : DBTestUniversalCompactionBase(
"/db_universal_compaction_multi_levels_test") {} "/db_universal_compaction_multi_levels_test") {}
}; };
TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) { TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
@ -725,12 +719,11 @@ INSTANTIATE_TEST_CASE_P(MultiLevels, DBTestUniversalCompactionMultiLevels,
::testing::Combine(::testing::Values(3, 20), ::testing::Combine(::testing::Values(3, 20),
::testing::Bool())); ::testing::Bool()));
class DBTestUniversalCompactionParallel : class DBTestUniversalCompactionParallel : public DBTestUniversalCompactionBase {
public DBTestUniversalCompactionBase {
public: public:
DBTestUniversalCompactionParallel() : DBTestUniversalCompactionParallel()
DBTestUniversalCompactionBase( : DBTestUniversalCompactionBase("/db_universal_compaction_prallel_test") {
"/db_universal_compaction_prallel_test") {} }
}; };
TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) { TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
@ -919,8 +912,8 @@ INSTANTIATE_TEST_CASE_P(Parallel, DBTestUniversalCompactionParallel,
TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) { TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.write_buffer_size = 105 << 10; // 105KB options.write_buffer_size = 105 << 10; // 105KB
options.arena_block_size = 4 << 10; // 4KB options.arena_block_size = 4 << 10; // 4KB
options.target_file_size_base = 32 << 10; // 32KB options.target_file_size_base = 32 << 10; // 32KB
options.level0_file_num_compaction_trigger = 4; options.level0_file_num_compaction_trigger = 4;
options.num_levels = num_levels_; options.num_levels = num_levels_;
@ -951,8 +944,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) { TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.write_buffer_size = 105 << 10; // 105KB options.write_buffer_size = 105 << 10; // 105KB
options.arena_block_size = 4 << 10; // 4KB options.arena_block_size = 4 << 10; // 4KB
options.target_file_size_base = 32 << 10; // 32KB options.target_file_size_base = 32 << 10; // 32KB
// trigger compaction if there are >= 4 files // trigger compaction if there are >= 4 files
options.level0_file_num_compaction_trigger = 4; options.level0_file_num_compaction_trigger = 4;
@ -1353,7 +1346,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCFPathUse) {
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 500 * 1024); cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 500 * 1024);
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_4", 1024 * 1024 * 1024); cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_4", 1024 * 1024 * 1024);
option_vector.emplace_back(DBOptions(options), cf_opt1); option_vector.emplace_back(DBOptions(options), cf_opt1);
CreateColumnFamilies({"one"},option_vector[1]); CreateColumnFamilies({"one"}, option_vector[1]);
// Configura CF2 specific paths. // Configura CF2 specific paths.
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 300 * 1024); cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 300 * 1024);
@ -1361,7 +1354,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCFPathUse) {
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 500 * 1024); cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 500 * 1024);
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_4", 1024 * 1024 * 1024); cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_4", 1024 * 1024 * 1024);
option_vector.emplace_back(DBOptions(options), cf_opt2); option_vector.emplace_back(DBOptions(options), cf_opt2);
CreateColumnFamilies({"two"},option_vector[2]); CreateColumnFamilies({"two"}, option_vector[2]);
ReopenWithColumnFamilies({"default", "one", "two"}, option_vector); ReopenWithColumnFamilies({"default", "one", "two"}, option_vector);
@ -1567,7 +1560,6 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
verify_func(max_key3); verify_func(max_key3);
} }
TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) { TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
if (!Snappy_Supported()) { if (!Snappy_Supported()) {
return; return;
@ -1829,9 +1821,9 @@ INSTANTIATE_TEST_CASE_P(NumLevels, DBTestUniversalCompaction,
class DBTestUniversalManualCompactionOutputPathId class DBTestUniversalManualCompactionOutputPathId
: public DBTestUniversalCompactionBase { : public DBTestUniversalCompactionBase {
public: public:
DBTestUniversalManualCompactionOutputPathId() : DBTestUniversalManualCompactionOutputPathId()
DBTestUniversalCompactionBase( : DBTestUniversalCompactionBase(
"/db_universal_compaction_manual_pid_test") {} "/db_universal_compaction_manual_pid_test") {}
}; };
TEST_P(DBTestUniversalManualCompactionOutputPathId, TEST_P(DBTestUniversalManualCompactionOutputPathId,
@ -2236,8 +2228,8 @@ int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
#else #else
(void) argc; (void)argc;
(void) argv; (void)argv;
return 0; return 0;
#endif #endif
} }

@ -170,7 +170,8 @@ TEST_P(DBWriteTest, WriteStallRemoveNoSlowdownWrite) {
TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) { TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
Options options = GetOptions(); Options options = GetOptions();
options.level0_stop_writes_trigger = options.level0_slowdown_writes_trigger = 4; options.level0_stop_writes_trigger = options.level0_slowdown_writes_trigger =
4;
std::vector<port::Thread> threads; std::vector<port::Thread> threads;
std::atomic<int> thread_num(0); std::atomic<int> thread_num(0);
port::Mutex mutex; port::Mutex mutex;
@ -195,7 +196,7 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
Status s = dbfull()->Put(wo, key, "bar"); Status s = dbfull()->Put(wo, key, "bar");
ASSERT_TRUE(s.ok() || s.IsIncomplete()); ASSERT_TRUE(s.ok() || s.IsIncomplete());
}; };
std::function<void(void *)> unblock_main_thread_func = [&](void *) { std::function<void(void*)> unblock_main_thread_func = [&](void*) {
mutex.Lock(); mutex.Lock();
++writers; ++writers;
cv.SignalAll(); cv.SignalAll();
@ -254,8 +255,9 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(nullptr)); ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(nullptr));
// This would have triggered a write stall. Unblock the write group leader // This would have triggered a write stall. Unblock the write group leader
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:2"); TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:2");
// The leader is going to create missing newer links. When the leader finishes, // The leader is going to create missing newer links. When the leader
// the next leader is going to delay writes and fail writers with no_slowdown // finishes, the next leader is going to delay writes and fail writers with
// no_slowdown
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:3"); TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:3");
for (auto& t : threads) { for (auto& t : threads) {
@ -623,42 +625,43 @@ TEST_P(DBWriteTest, LockWalInEffect) {
} }
TEST_P(DBWriteTest, ConcurrentlyDisabledWAL) { TEST_P(DBWriteTest, ConcurrentlyDisabledWAL) {
Options options = GetOptions(); Options options = GetOptions();
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
options.statistics->set_stats_level(StatsLevel::kAll); options.statistics->set_stats_level(StatsLevel::kAll);
Reopen(options); Reopen(options);
std::string wal_key_prefix = "WAL_KEY_"; std::string wal_key_prefix = "WAL_KEY_";
std::string no_wal_key_prefix = "K_"; std::string no_wal_key_prefix = "K_";
// 100 KB value each for NO-WAL operation // 100 KB value each for NO-WAL operation
std::string no_wal_value(1024 * 100, 'X'); std::string no_wal_value(1024 * 100, 'X');
// 1B value each for WAL operation // 1B value each for WAL operation
std::string wal_value = "0"; std::string wal_value = "0";
std::thread threads[10]; std::thread threads[10];
for (int t = 0; t < 10; t++) { for (int t = 0; t < 10; t++) {
threads[t] = std::thread([t, wal_key_prefix, wal_value, no_wal_key_prefix, no_wal_value, this] { threads[t] = std::thread([t, wal_key_prefix, wal_value, no_wal_key_prefix,
for(int i = 0; i < 10; i++) { no_wal_value, this] {
ROCKSDB_NAMESPACE::WriteOptions write_option_disable; for (int i = 0; i < 10; i++) {
write_option_disable.disableWAL = true; ROCKSDB_NAMESPACE::WriteOptions write_option_disable;
ROCKSDB_NAMESPACE::WriteOptions write_option_default; write_option_disable.disableWAL = true;
std::string no_wal_key = no_wal_key_prefix + std::to_string(t) + ROCKSDB_NAMESPACE::WriteOptions write_option_default;
"_" + std::to_string(i); std::string no_wal_key =
ASSERT_OK( no_wal_key_prefix + std::to_string(t) + "_" + std::to_string(i);
this->Put(no_wal_key, no_wal_value, write_option_disable)); ASSERT_OK(this->Put(no_wal_key, no_wal_value, write_option_disable));
std::string wal_key = std::string wal_key =
wal_key_prefix + std::to_string(i) + "_" + std::to_string(i); wal_key_prefix + std::to_string(i) + "_" + std::to_string(i);
ASSERT_OK(this->Put(wal_key, wal_value, write_option_default)); ASSERT_OK(this->Put(wal_key, wal_value, write_option_default));
ASSERT_OK(dbfull()->SyncWAL()); ASSERT_OK(dbfull()->SyncWAL());
} }
return; return;
}); });
} }
for (auto& t: threads) { for (auto& t : threads) {
t.join(); t.join();
} }
uint64_t bytes_num = options.statistics->getTickerCount( uint64_t bytes_num = options.statistics->getTickerCount(
ROCKSDB_NAMESPACE::Tickers::WAL_FILE_BYTES); ROCKSDB_NAMESPACE::Tickers::WAL_FILE_BYTES);
// written WAL size should less than 100KB (even included HEADER & FOOTER overhead) // written WAL size should less than 100KB (even included HEADER & FOOTER
ASSERT_LE(bytes_num, 1024 * 100); // overhead)
ASSERT_LE(bytes_num, 1024 * 100);
} }
INSTANTIATE_TEST_CASE_P(DBWriteTestInstance, DBWriteTest, INSTANTIATE_TEST_CASE_P(DBWriteTestInstance, DBWriteTest,

@ -15,8 +15,7 @@
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
static std::string IKey(const std::string& user_key, static std::string IKey(const std::string& user_key, uint64_t seq,
uint64_t seq,
ValueType vt) { ValueType vt) {
std::string encoded; std::string encoded;
AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt)); AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
@ -37,9 +36,7 @@ static std::string ShortSuccessor(const std::string& s) {
return result; return result;
} }
static void TestKey(const std::string& key, static void TestKey(const std::string& key, uint64_t seq, ValueType vt) {
uint64_t seq,
ValueType vt) {
std::string encoded = IKey(key, seq, vt); std::string encoded = IKey(key, seq, vt);
Slice in(encoded); Slice in(encoded);
@ -56,13 +53,19 @@ static void TestKey(const std::string& key,
class FormatTest : public testing::Test {}; class FormatTest : public testing::Test {};
TEST_F(FormatTest, InternalKey_EncodeDecode) { TEST_F(FormatTest, InternalKey_EncodeDecode) {
const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" }; const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"};
const uint64_t seq[] = { const uint64_t seq[] = {1,
1, 2, 3, 2,
(1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1, 3,
(1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1, (1ull << 8) - 1,
(1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1 1ull << 8,
}; (1ull << 8) + 1,
(1ull << 16) - 1,
1ull << 16,
(1ull << 16) + 1,
(1ull << 32) - 1,
1ull << 32,
(1ull << 32) + 1};
for (unsigned int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) { for (unsigned int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
for (unsigned int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) { for (unsigned int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
TestKey(keys[k], seq[s], kTypeValue); TestKey(keys[k], seq[s], kTypeValue);
@ -74,27 +77,25 @@ TEST_F(FormatTest, InternalKey_EncodeDecode) {
TEST_F(FormatTest, InternalKeyShortSeparator) { TEST_F(FormatTest, InternalKeyShortSeparator) {
// When user keys are same // When user keys are same
ASSERT_EQ(IKey("foo", 100, kTypeValue), ASSERT_EQ(IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue), Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue)));
IKey("foo", 99, kTypeValue))); ASSERT_EQ(
ASSERT_EQ(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue), Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue)));
IKey("foo", 101, kTypeValue))); ASSERT_EQ(
ASSERT_EQ(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue), Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue)));
IKey("foo", 100, kTypeValue))); ASSERT_EQ(
ASSERT_EQ(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue), Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion)));
IKey("foo", 100, kTypeDeletion)));
// When user keys are misordered // When user keys are misordered
ASSERT_EQ(IKey("foo", 100, kTypeValue), ASSERT_EQ(IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue), Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue)));
IKey("bar", 99, kTypeValue)));
// When user keys are different, but correctly ordered // When user keys are different, but correctly ordered
ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), ASSERT_EQ(
Shorten(IKey("foo", 100, kTypeValue), IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
IKey("hello", 200, kTypeValue))); Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue)));
ASSERT_EQ(IKey("ABC2", kMaxSequenceNumber, kValueTypeForSeek), ASSERT_EQ(IKey("ABC2", kMaxSequenceNumber, kValueTypeForSeek),
Shorten(IKey("ABC1AAAAA", 100, kTypeValue), Shorten(IKey("ABC1AAAAA", 100, kTypeValue),
@ -121,14 +122,14 @@ TEST_F(FormatTest, InternalKeyShortSeparator) {
Shorten(IKey("AAA1", 100, kTypeValue), IKey("AAA2", 200, kTypeValue))); Shorten(IKey("AAA1", 100, kTypeValue), IKey("AAA2", 200, kTypeValue)));
// When start user key is prefix of limit user key // When start user key is prefix of limit user key
ASSERT_EQ(IKey("foo", 100, kTypeValue), ASSERT_EQ(
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue),
IKey("foobar", 200, kTypeValue))); Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue)));
// When limit user key is prefix of start user key // When limit user key is prefix of start user key
ASSERT_EQ(IKey("foobar", 100, kTypeValue), ASSERT_EQ(
Shorten(IKey("foobar", 100, kTypeValue), IKey("foobar", 100, kTypeValue),
IKey("foo", 200, kTypeValue))); Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue)));
} }
TEST_F(FormatTest, InternalKeyShortestSuccessor) { TEST_F(FormatTest, InternalKeyShortestSuccessor) {

@ -10,9 +10,11 @@
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#include <stdlib.h> #include <stdlib.h>
#include <map> #include <map>
#include <string> #include <string>
#include <vector> #include <vector>
#include "db/db_impl/db_impl.h" #include "db/db_impl/db_impl.h"
#include "db/db_test_util.h" #include "db/db_test_util.h"
#include "db/version_set.h" #include "db/version_set.h"
@ -55,7 +57,7 @@ class DeleteFileTest : public DBTestBase {
WriteOptions options; WriteOptions options;
options.sync = false; options.sync = false;
ReadOptions roptions; ReadOptions roptions;
for (int i = startkey; i < (numkeys + startkey) ; i++) { for (int i = startkey; i < (numkeys + startkey); i++) {
std::string temp = std::to_string(i); std::string temp = std::to_string(i);
Slice key(temp); Slice key(temp);
Slice value(temp); Slice value(temp);
@ -63,10 +65,8 @@ class DeleteFileTest : public DBTestBase {
} }
} }
int numKeysInLevels( int numKeysInLevels(std::vector<LiveFileMetaData>& metadata,
std::vector<LiveFileMetaData> &metadata, std::vector<int>* keysperlevel = nullptr) {
std::vector<int> *keysperlevel = nullptr) {
if (keysperlevel != nullptr) { if (keysperlevel != nullptr) {
keysperlevel->resize(numlevels_); keysperlevel->resize(numlevels_);
} }
@ -82,8 +82,7 @@ class DeleteFileTest : public DBTestBase {
} }
fprintf(stderr, "level %d name %s smallest %s largest %s\n", fprintf(stderr, "level %d name %s smallest %s largest %s\n",
metadata[i].level, metadata[i].name.c_str(), metadata[i].level, metadata[i].name.c_str(),
metadata[i].smallestkey.c_str(), metadata[i].smallestkey.c_str(), metadata[i].largestkey.c_str());
metadata[i].largestkey.c_str());
} }
return numKeys; return numKeys;
} }
@ -214,7 +213,7 @@ TEST_F(DeleteFileTest, PurgeObsoleteFilesTest) {
// this time, we keep an iterator alive // this time, we keep an iterator alive
Reopen(options); Reopen(options);
Iterator *itr = nullptr; Iterator* itr = nullptr;
CreateTwoLevels(); CreateTwoLevels();
itr = db_->NewIterator(ReadOptions()); itr = db_->NewIterator(ReadOptions());
ASSERT_OK(itr->status()); ASSERT_OK(itr->status());
@ -481,12 +480,12 @@ TEST_F(DeleteFileTest, DeleteFileWithIterator) {
} }
Status status = db_->DeleteFile(level2file); Status status = db_->DeleteFile(level2file);
fprintf(stdout, "Deletion status %s: %s\n", fprintf(stdout, "Deletion status %s: %s\n", level2file.c_str(),
level2file.c_str(), status.ToString().c_str()); status.ToString().c_str());
ASSERT_OK(status); ASSERT_OK(status);
it->SeekToFirst(); it->SeekToFirst();
int numKeysIterated = 0; int numKeysIterated = 0;
while(it->Valid()) { while (it->Valid()) {
numKeysIterated++; numKeysIterated++;
it->Next(); it->Next();
} }

@ -234,8 +234,8 @@ void ErrorHandler::CancelErrorRecovery() {
// We'll release the lock before calling sfm, so make sure no new // We'll release the lock before calling sfm, so make sure no new
// recovery gets scheduled at that point // recovery gets scheduled at that point
auto_recovery_ = false; auto_recovery_ = false;
SstFileManagerImpl* sfm = reinterpret_cast<SstFileManagerImpl*>( SstFileManagerImpl* sfm =
db_options_.sst_file_manager.get()); reinterpret_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
if (sfm) { if (sfm) {
// This may or may not cancel a pending recovery // This may or may not cancel a pending recovery
db_mutex_->Unlock(); db_mutex_->Unlock();
@ -292,8 +292,8 @@ const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err,
bool found = false; bool found = false;
{ {
auto entry = ErrorSeverityMap.find(std::make_tuple(reason, bg_err.code(), auto entry = ErrorSeverityMap.find(
bg_err.subcode(), paranoid)); std::make_tuple(reason, bg_err.code(), bg_err.subcode(), paranoid));
if (entry != ErrorSeverityMap.end()) { if (entry != ErrorSeverityMap.end()) {
sev = entry->second; sev = entry->second;
found = true; found = true;
@ -301,8 +301,8 @@ const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err,
} }
if (!found) { if (!found) {
auto entry = DefaultErrorSeverityMap.find(std::make_tuple(reason, auto entry = DefaultErrorSeverityMap.find(
bg_err.code(), paranoid)); std::make_tuple(reason, bg_err.code(), paranoid));
if (entry != DefaultErrorSeverityMap.end()) { if (entry != DefaultErrorSeverityMap.end()) {
sev = entry->second; sev = entry->second;
found = true; found = true;

@ -26,100 +26,99 @@ struct DBRecoverContext {
}; };
class ErrorHandler { class ErrorHandler {
public: public:
ErrorHandler(DBImpl* db, const ImmutableDBOptions& db_options, ErrorHandler(DBImpl* db, const ImmutableDBOptions& db_options,
InstrumentedMutex* db_mutex) InstrumentedMutex* db_mutex)
: db_(db), : db_(db),
db_options_(db_options), db_options_(db_options),
cv_(db_mutex), cv_(db_mutex),
end_recovery_(false), end_recovery_(false),
recovery_thread_(nullptr), recovery_thread_(nullptr),
db_mutex_(db_mutex), db_mutex_(db_mutex),
auto_recovery_(false), auto_recovery_(false),
recovery_in_prog_(false), recovery_in_prog_(false),
soft_error_no_bg_work_(false), soft_error_no_bg_work_(false),
is_db_stopped_(false), is_db_stopped_(false),
bg_error_stats_(db_options.statistics) { bg_error_stats_(db_options.statistics) {
// Clear the checked flag for uninitialized errors // Clear the checked flag for uninitialized errors
bg_error_.PermitUncheckedError(); bg_error_.PermitUncheckedError();
recovery_error_.PermitUncheckedError(); recovery_error_.PermitUncheckedError();
recovery_io_error_.PermitUncheckedError(); recovery_io_error_.PermitUncheckedError();
} }
void EnableAutoRecovery() { auto_recovery_ = true; } void EnableAutoRecovery() { auto_recovery_ = true; }
Status::Severity GetErrorSeverity(BackgroundErrorReason reason, Status::Severity GetErrorSeverity(BackgroundErrorReason reason,
Status::Code code, Status::Code code, Status::SubCode subcode);
Status::SubCode subcode);
const Status& SetBGError(const Status& bg_err, BackgroundErrorReason reason);
const Status& SetBGError(const Status& bg_err, BackgroundErrorReason reason);
Status GetBGError() const { return bg_error_; }
Status GetBGError() const { return bg_error_; }
Status GetRecoveryError() const { return recovery_error_; }
Status GetRecoveryError() const { return recovery_error_; }
Status ClearBGError();
Status ClearBGError();
bool IsDBStopped() { return is_db_stopped_.load(std::memory_order_acquire); }
bool IsDBStopped() { return is_db_stopped_.load(std::memory_order_acquire); }
bool IsBGWorkStopped() {
bool IsBGWorkStopped() { assert(db_mutex_);
assert(db_mutex_); db_mutex_->AssertHeld();
db_mutex_->AssertHeld(); return !bg_error_.ok() &&
return !bg_error_.ok() && (bg_error_.severity() >= Status::Severity::kHardError ||
(bg_error_.severity() >= Status::Severity::kHardError || !auto_recovery_ || soft_error_no_bg_work_);
!auto_recovery_ || soft_error_no_bg_work_); }
}
bool IsSoftErrorNoBGWork() { return soft_error_no_bg_work_; }
bool IsSoftErrorNoBGWork() { return soft_error_no_bg_work_; }
bool IsRecoveryInProgress() { return recovery_in_prog_; }
bool IsRecoveryInProgress() { return recovery_in_prog_; }
Status RecoverFromBGError(bool is_manual = false);
Status RecoverFromBGError(bool is_manual = false); void CancelErrorRecovery();
void CancelErrorRecovery();
void EndAutoRecovery();
void EndAutoRecovery();
private:
private: DBImpl* db_;
DBImpl* db_; const ImmutableDBOptions& db_options_;
const ImmutableDBOptions& db_options_; Status bg_error_;
Status bg_error_; // A separate Status variable used to record any errors during the
// A separate Status variable used to record any errors during the // recovery process from hard errors
// recovery process from hard errors Status recovery_error_;
Status recovery_error_; // A separate IO Status variable used to record any IO errors during
// A separate IO Status variable used to record any IO errors during // the recovery process. At the same time, recovery_error_ is also set.
// the recovery process. At the same time, recovery_error_ is also set. IOStatus recovery_io_error_;
IOStatus recovery_io_error_; // The condition variable used with db_mutex during auto resume for time
// The condition variable used with db_mutex during auto resume for time // wait.
// wait. InstrumentedCondVar cv_;
InstrumentedCondVar cv_; bool end_recovery_;
bool end_recovery_; std::unique_ptr<port::Thread> recovery_thread_;
std::unique_ptr<port::Thread> recovery_thread_;
InstrumentedMutex* db_mutex_;
InstrumentedMutex* db_mutex_; // A flag indicating whether automatic recovery from errors is enabled
// A flag indicating whether automatic recovery from errors is enabled bool auto_recovery_;
bool auto_recovery_; bool recovery_in_prog_;
bool recovery_in_prog_; // A flag to indicate that for the soft error, we should not allow any
// A flag to indicate that for the soft error, we should not allow any // background work except the work is from recovery.
// background work except the work is from recovery. bool soft_error_no_bg_work_;
bool soft_error_no_bg_work_;
// Used to store the context for recover, such as flush reason.
// Used to store the context for recover, such as flush reason. DBRecoverContext recover_context_;
DBRecoverContext recover_context_; std::atomic<bool> is_db_stopped_;
std::atomic<bool> is_db_stopped_;
// The pointer of DB statistics.
// The pointer of DB statistics. std::shared_ptr<Statistics> bg_error_stats_;
std::shared_ptr<Statistics> bg_error_stats_;
const Status& HandleKnownErrors(const Status& bg_err,
const Status& HandleKnownErrors(const Status& bg_err, BackgroundErrorReason reason);
BackgroundErrorReason reason); Status OverrideNoSpaceError(const Status& bg_error, bool* auto_recovery);
Status OverrideNoSpaceError(const Status& bg_error, bool* auto_recovery); void RecoverFromNoSpace();
void RecoverFromNoSpace(); const Status& StartRecoverFromRetryableBGIOError(const IOStatus& io_error);
const Status& StartRecoverFromRetryableBGIOError(const IOStatus& io_error); void RecoverFromRetryableBGIOError();
void RecoverFromRetryableBGIOError(); // First, if it is in recovery and the recovery_error is ok. Set the
// First, if it is in recovery and the recovery_error is ok. Set the // recovery_error_ to bg_err. Second, if the severity is higher than the
// recovery_error_ to bg_err. Second, if the severity is higher than the // current bg_error_, overwrite it.
// current bg_error_, overwrite it. void CheckAndSetRecoveryAndBGError(const Status& bg_err);
void CheckAndSetRecoveryAndBGError(const Status& bg_err);
}; };
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -23,7 +23,7 @@ template <class T>
inline T SafeDivide(T a, T b) { inline T SafeDivide(T a, T b) {
return b == 0 ? 0 : a / b; return b == 0 ? 0 : a / b;
} }
} // namespace } // anonymous namespace
void EventHelpers::AppendCurrentTime(JSONWriter* jwriter) { void EventHelpers::AppendCurrentTime(JSONWriter* jwriter) {
*jwriter << "time_micros" *jwriter << "time_micros"

@ -39,9 +39,9 @@ class EventHelpers {
const std::string& file_checksum, const std::string& file_checksum,
const std::string& file_checksum_func_name); const std::string& file_checksum_func_name);
static void LogAndNotifyTableFileDeletion( static void LogAndNotifyTableFileDeletion(
EventLogger* event_logger, int job_id, EventLogger* event_logger, int job_id, uint64_t file_number,
uint64_t file_number, const std::string& file_path, const std::string& file_path, const Status& status,
const Status& status, const std::string& db_name, const std::string& db_name,
const std::vector<std::shared_ptr<EventListener>>& listeners); const std::vector<std::shared_ptr<EventListener>>& listeners);
static void NotifyOnErrorRecoveryEnd( static void NotifyOnErrorRecoveryEnd(
const std::vector<std::shared_ptr<EventListener>>& listeners, const std::vector<std::shared_ptr<EventListener>>& listeners,

@ -106,9 +106,8 @@ Status ExternalSstFileIngestionJob::Prepare(
for (IngestedFileInfo& f : files_to_ingest_) { for (IngestedFileInfo& f : files_to_ingest_) {
f.copy_file = false; f.copy_file = false;
const std::string path_outside_db = f.external_file_path; const std::string path_outside_db = f.external_file_path;
const std::string path_inside_db = const std::string path_inside_db = TableFileName(
TableFileName(cfd_->ioptions()->cf_paths, f.fd.GetNumber(), cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId());
f.fd.GetPathId());
if (ingestion_options_.move_files) { if (ingestion_options_.move_files) {
status = status =
fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr); fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr);
@ -491,7 +490,8 @@ void ExternalSstFileIngestionJob::UpdateStats() {
stream.StartArray(); stream.StartArray();
for (IngestedFileInfo& f : files_to_ingest_) { for (IngestedFileInfo& f : files_to_ingest_) {
InternalStats::CompactionStats stats(CompactionReason::kExternalSstIngestion, 1); InternalStats::CompactionStats stats(
CompactionReason::kExternalSstIngestion, 1);
stats.micros = total_time; stats.micros = total_time;
// If actual copy occurred for this file, then we need to count the file // If actual copy occurred for this file, then we need to count the file
// size as the actual bytes written. If the file was linked, then we ignore // size as the actual bytes written. If the file was linked, then we ignore
@ -591,8 +591,8 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
std::unique_ptr<FSRandomAccessFile> sst_file; std::unique_ptr<FSRandomAccessFile> sst_file;
std::unique_ptr<RandomAccessFileReader> sst_file_reader; std::unique_ptr<RandomAccessFileReader> sst_file_reader;
status = fs_->NewRandomAccessFile(external_file, env_options_, status =
&sst_file, nullptr); fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
@ -658,9 +658,9 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
assert(seqno_iter == uprops.end()); assert(seqno_iter == uprops.end());
file_to_ingest->original_seqno = 0; file_to_ingest->original_seqno = 0;
if (ingestion_options_.allow_blocking_flush || if (ingestion_options_.allow_blocking_flush ||
ingestion_options_.allow_global_seqno) { ingestion_options_.allow_global_seqno) {
return Status::InvalidArgument( return Status::InvalidArgument(
"External SST file V1 does not support global seqno"); "External SST file V1 does not support global seqno");
} }
} else { } else {
return Status::InvalidArgument("External file version is not supported"); return Status::InvalidArgument("External file version is not supported");
@ -857,7 +857,7 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
return status; return status;
} }
TEST_SYNC_POINT_CALLBACK( TEST_SYNC_POINT_CALLBACK(
"ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile", "ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile",
&overlap_with_db); &overlap_with_db);
file_to_ingest->picked_level = target_level; file_to_ingest->picked_level = target_level;
@ -872,10 +872,10 @@ Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
auto* vstorage = cfd_->current()->storage_info(); auto* vstorage = cfd_->current()->storage_info();
// first check if new files fit in the bottommost level // first check if new files fit in the bottommost level
int bottom_lvl = cfd_->NumberLevels() - 1; int bottom_lvl = cfd_->NumberLevels() - 1;
if(!IngestedFileFitInLevel(file_to_ingest, bottom_lvl)) { if (!IngestedFileFitInLevel(file_to_ingest, bottom_lvl)) {
return Status::InvalidArgument( return Status::InvalidArgument(
"Can't ingest_behind file as it doesn't fit " "Can't ingest_behind file as it doesn't fit "
"at the bottommost level!"); "at the bottommost level!");
} }
// second check if despite allow_ingest_behind=true we still have 0 seqnums // second check if despite allow_ingest_behind=true we still have 0 seqnums
@ -884,8 +884,8 @@ Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
for (auto file : vstorage->LevelFiles(lvl)) { for (auto file : vstorage->LevelFiles(lvl)) {
if (file->fd.smallest_seqno == 0) { if (file->fd.smallest_seqno == 0) {
return Status::InvalidArgument( return Status::InvalidArgument(
"Can't ingest_behind file as despite allow_ingest_behind=true " "Can't ingest_behind file as despite allow_ingest_behind=true "
"there are files with 0 seqno in database at upper levels!"); "there are files with 0 seqno in database at upper levels!");
} }
} }
} }
@ -912,9 +912,8 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
// If the file system does not support random write, then we should not. // If the file system does not support random write, then we should not.
// Otherwise we should. // Otherwise we should.
std::unique_ptr<FSRandomRWFile> rwfile; std::unique_ptr<FSRandomRWFile> rwfile;
Status status = Status status = fs_->NewRandomRWFile(file_to_ingest->internal_file_path,
fs_->NewRandomRWFile(file_to_ingest->internal_file_path, env_options_, env_options_, &rwfile, nullptr);
&rwfile, nullptr);
TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::NewRandomRWFile", TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::NewRandomRWFile",
&status); &status);
if (status.ok()) { if (status.ok()) {

@ -301,7 +301,8 @@ TEST_F(ExternalSSTFileTest, Basic) {
SstFileWriter sst_file_writer(EnvOptions(), options); SstFileWriter sst_file_writer(EnvOptions(), options);
// Current file size should be 0 after sst_file_writer init and before open a file. // Current file size should be 0 after sst_file_writer init and before open
// a file.
ASSERT_EQ(sst_file_writer.FileSize(), 0); ASSERT_EQ(sst_file_writer.FileSize(), 0);
// file1.sst (0 => 99) // file1.sst (0 => 99)
@ -2318,7 +2319,6 @@ TEST_F(ExternalSSTFileTest, SkipBloomFilter) {
table_options.cache_index_and_filter_blocks = true; table_options.cache_index_and_filter_blocks = true;
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
// Create external SST file and include bloom filters // Create external SST file and include bloom filters
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
DestroyAndReopen(options); DestroyAndReopen(options);

@ -338,8 +338,7 @@ class FaultInjectionTest
FaultInjectionTest::kValExpectNoError)); FaultInjectionTest::kValExpectNoError));
} }
void NoWriteTestPreFault() { void NoWriteTestPreFault() {}
}
void NoWriteTestReopenWithFault(ResetMethod reset_method) { void NoWriteTestReopenWithFault(ResetMethod reset_method) {
CloseDB(); CloseDB();

@ -8,8 +8,10 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/file_indexer.h" #include "db/file_indexer.h"
#include <algorithm> #include <algorithm>
#include <functional> #include <functional>
#include "db/version_edit.h" #include "db/version_edit.h"
#include "rocksdb/comparator.h" #include "rocksdb/comparator.h"

@ -12,6 +12,7 @@
#include <functional> #include <functional>
#include <limits> #include <limits>
#include <vector> #include <vector>
#include "memory/arena.h" #include "memory/arena.h"
#include "port/port.h" #include "port/port.h"
#include "util/autovector.h" #include "util/autovector.h"
@ -66,7 +67,7 @@ class FileIndexer {
struct IndexUnit { struct IndexUnit {
IndexUnit() IndexUnit()
: smallest_lb(0), largest_lb(0), smallest_rb(-1), largest_rb(-1) {} : smallest_lb(0), largest_lb(0), smallest_rb(-1), largest_rb(-1) {}
// During file search, a key is compared against smallest and largest // During file search, a key is compared against smallest and largest
// from a FileMetaData. It can have 3 possible outcomes: // from a FileMetaData. It can have 3 possible outcomes:
// (1) key is smaller than smallest, implying it is also smaller than // (1) key is smaller than smallest, implying it is also smaller than

@ -8,7 +8,9 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/file_indexer.h" #include "db/file_indexer.h"
#include <string> #include <string>
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/version_edit.h" #include "db/version_edit.h"
#include "port/stack_trace.h" #include "port/stack_trace.h"
@ -73,8 +75,8 @@ class FileIndexerTest : public testing::Test {
} }
void GetNextLevelIndex(const uint32_t level, const uint32_t file_index, void GetNextLevelIndex(const uint32_t level, const uint32_t file_index,
const int cmp_smallest, const int cmp_largest, int32_t* left_index, const int cmp_smallest, const int cmp_largest,
int32_t* right_index) { int32_t* left_index, int32_t* right_index) {
*left_index = 100; *left_index = 100;
*right_index = 100; *right_index = 100;
indexer->GetNextLevelIndex(level, file_index, cmp_smallest, cmp_largest, indexer->GetNextLevelIndex(level, file_index, cmp_smallest, cmp_largest,

@ -69,35 +69,33 @@ TEST_F(FileNameTest, Parse) {
} }
// Errors // Errors
static const char* errors[] = { static const char* errors[] = {"",
"", "foo",
"foo", "foo-dx-100.log",
"foo-dx-100.log", ".log",
".log", "",
"", "manifest",
"manifest", "CURREN",
"CURREN", "CURRENTX",
"CURRENTX", "MANIFES",
"MANIFES", "MANIFEST",
"MANIFEST", "MANIFEST-",
"MANIFEST-", "XMANIFEST-3",
"XMANIFEST-3", "MANIFEST-3x",
"MANIFEST-3x", "META",
"META", "METADB",
"METADB", "METADB-",
"METADB-", "XMETADB-3",
"XMETADB-3", "METADB-3x",
"METADB-3x", "LOC",
"LOC", "LOCKx",
"LOCKx", "LO",
"LO", "LOGx",
"LOGx", "18446744073709551616.log",
"18446744073709551616.log", "184467440737095516150.log",
"184467440737095516150.log", "100",
"100", "100.",
"100.", "100.lop"};
"100.lop"
};
for (unsigned int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { for (unsigned int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
std::string f = errors[i]; std::string f = errors[i];
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;

@ -48,7 +48,7 @@
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
const char* GetFlushReasonString (FlushReason flush_reason) { const char* GetFlushReasonString(FlushReason flush_reason) {
switch (flush_reason) { switch (flush_reason) {
case FlushReason::kOthers: case FlushReason::kOthers:
return "Other Reasons"; return "Other Reasons";
@ -136,17 +136,14 @@ FlushJob::FlushJob(
TEST_SYNC_POINT("FlushJob::FlushJob()"); TEST_SYNC_POINT("FlushJob::FlushJob()");
} }
FlushJob::~FlushJob() { FlushJob::~FlushJob() { ThreadStatusUtil::ResetThreadStatus(); }
ThreadStatusUtil::ResetThreadStatus();
}
void FlushJob::ReportStartedFlush() { void FlushJob::ReportStartedFlush() {
ThreadStatusUtil::SetColumnFamily(cfd_, cfd_->ioptions()->env, ThreadStatusUtil::SetColumnFamily(cfd_, cfd_->ioptions()->env,
db_options_.enable_thread_tracking); db_options_.enable_thread_tracking);
ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH); ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH);
ThreadStatusUtil::SetThreadOperationProperty( ThreadStatusUtil::SetThreadOperationProperty(ThreadStatus::COMPACTION_JOB_ID,
ThreadStatus::COMPACTION_JOB_ID, job_context_->job_id);
job_context_->job_id);
IOSTATS_RESET(bytes_written); IOSTATS_RESET(bytes_written);
} }
@ -156,8 +153,7 @@ void FlushJob::ReportFlushInputSize(const autovector<MemTable*>& mems) {
input_size += mem->ApproximateMemoryUsage(); input_size += mem->ApproximateMemoryUsage();
} }
ThreadStatusUtil::IncreaseThreadOperationProperty( ThreadStatusUtil::IncreaseThreadOperationProperty(
ThreadStatus::FLUSH_BYTES_MEMTABLES, ThreadStatus::FLUSH_BYTES_MEMTABLES, input_size);
input_size);
} }
void FlushJob::RecordFlushIOStats() { void FlushJob::RecordFlushIOStats() {
@ -220,8 +216,7 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta,
double mempurge_threshold = double mempurge_threshold =
mutable_cf_options_.experimental_mempurge_threshold; mutable_cf_options_.experimental_mempurge_threshold;
AutoThreadOperationStageUpdater stage_run( AutoThreadOperationStageUpdater stage_run(ThreadStatus::STAGE_FLUSH_RUN);
ThreadStatus::STAGE_FLUSH_RUN);
if (mems_.empty()) { if (mems_.empty()) {
ROCKS_LOG_BUFFER(log_buffer_, "[%s] Nothing in memtable to flush", ROCKS_LOG_BUFFER(log_buffer_, "[%s] Nothing in memtable to flush",
cfd_->GetName().c_str()); cfd_->GetName().c_str());
@ -906,8 +901,7 @@ Status FlushJob::WriteLevel0Table() {
} }
const uint64_t current_time = static_cast<uint64_t>(_current_time); const uint64_t current_time = static_cast<uint64_t>(_current_time);
uint64_t oldest_key_time = uint64_t oldest_key_time = mems_.front()->ApproximateOldestKeyTime();
mems_.front()->ApproximateOldestKeyTime();
// It's not clear whether oldest_key_time is always available. In case // It's not clear whether oldest_key_time is always available. In case
// it is not available, use current_time. // it is not available, use current_time.

@ -214,8 +214,8 @@ TEST_F(FlushJobTest, NonEmpty) {
// Note: the first two blob references will not be considered when resolving // Note: the first two blob references will not be considered when resolving
// the oldest blob file referenced (the first one is inlined TTL, while the // the oldest blob file referenced (the first one is inlined TTL, while the
// second one is TTL and thus points to a TTL blob file). // second one is TTL and thus points to a TTL blob file).
constexpr std::array<uint64_t, 6> blob_file_numbers{{ constexpr std::array<uint64_t, 6> blob_file_numbers{
kInvalidBlobFileNumber, 5, 103, 17, 102, 101}}; {kInvalidBlobFileNumber, 5, 103, 17, 102, 101}};
for (size_t i = 0; i < blob_file_numbers.size(); ++i) { for (size_t i = 0; i < blob_file_numbers.size(); ++i) {
std::string key(std::to_string(i + 10001)); std::string key(std::to_string(i + 10001));
std::string blob_index; std::string blob_index;

@ -104,9 +104,7 @@ class ForwardLevelIterator : public InternalIterator {
status_ = Status::NotSupported("ForwardLevelIterator::Prev()"); status_ = Status::NotSupported("ForwardLevelIterator::Prev()");
valid_ = false; valid_ = false;
} }
bool Valid() const override { bool Valid() const override { return valid_; }
return valid_;
}
void SeekToFirst() override { void SeekToFirst() override {
assert(file_iter_ != nullptr); assert(file_iter_ != nullptr);
if (!status_.ok()) { if (!status_.ok()) {
@ -249,9 +247,7 @@ ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
immutable_status_.PermitUncheckedError(); immutable_status_.PermitUncheckedError();
} }
ForwardIterator::~ForwardIterator() { ForwardIterator::~ForwardIterator() { Cleanup(true); }
Cleanup(true);
}
void ForwardIterator::SVCleanup(DBImpl* db, SuperVersion* sv, void ForwardIterator::SVCleanup(DBImpl* db, SuperVersion* sv,
bool background_purge_on_iterator_cleanup) { bool background_purge_on_iterator_cleanup) {
@ -284,13 +280,13 @@ struct SVCleanupParams {
SuperVersion* sv; SuperVersion* sv;
bool background_purge_on_iterator_cleanup; bool background_purge_on_iterator_cleanup;
}; };
} } // anonymous namespace
// Used in PinnedIteratorsManager to release pinned SuperVersion // Used in PinnedIteratorsManager to release pinned SuperVersion
void ForwardIterator::DeferredSVCleanup(void* arg) { void ForwardIterator::DeferredSVCleanup(void* arg) {
auto d = reinterpret_cast<SVCleanupParams*>(arg); auto d = reinterpret_cast<SVCleanupParams*>(arg);
ForwardIterator::SVCleanup( ForwardIterator::SVCleanup(d->db, d->sv,
d->db, d->sv, d->background_purge_on_iterator_cleanup); d->background_purge_on_iterator_cleanup);
delete d; delete d;
} }
@ -547,8 +543,7 @@ void ForwardIterator::Next() {
assert(valid_); assert(valid_);
bool update_prev_key = false; bool update_prev_key = false;
if (sv_ == nullptr || if (sv_ == nullptr || sv_->version_number != cfd_->GetSuperVersionNumber()) {
sv_->version_number != cfd_->GetSuperVersionNumber()) {
std::string current_key = key().ToString(); std::string current_key = key().ToString();
Slice old_key(current_key.data(), current_key.size()); Slice old_key(current_key.data(), current_key.size());
@ -578,7 +573,6 @@ void ForwardIterator::Next() {
update_prev_key = true; update_prev_key = true;
} }
if (update_prev_key) { if (update_prev_key) {
prev_key_.SetInternalKey(current_->key()); prev_key_.SetInternalKey(current_->key());
is_prev_set_ = true; is_prev_set_ = true;
@ -635,7 +629,7 @@ bool ForwardIterator::PrepareValue() {
assert(!current_->Valid()); assert(!current_->Valid());
assert(!current_->status().ok()); assert(!current_->status().ok());
assert(current_ != mutable_iter_); // memtable iterator can't fail assert(current_ != mutable_iter_); // memtable iterator can't fail
assert(immutable_status_.ok()); assert(immutable_status_.ok());
valid_ = false; valid_ = false;
@ -950,11 +944,11 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
} }
Slice prev_key = prev_key_.GetInternalKey(); Slice prev_key = prev_key_.GetInternalKey();
if (prefix_extractor_ && prefix_extractor_->Transform(target).compare( if (prefix_extractor_ && prefix_extractor_->Transform(target).compare(
prefix_extractor_->Transform(prev_key)) != 0) { prefix_extractor_->Transform(prev_key)) != 0) {
return true; return true;
} }
if (cfd_->internal_comparator().InternalKeyComparator::Compare( if (cfd_->internal_comparator().InternalKeyComparator::Compare(
prev_key, target) >= (is_prev_inclusive_ ? 1 : 0)) { prev_key, target) >= (is_prev_inclusive_ ? 1 : 0)) {
return true; return true;
} }
@ -963,8 +957,8 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
return false; return false;
} }
if (cfd_->internal_comparator().InternalKeyComparator::Compare( if (cfd_->internal_comparator().InternalKeyComparator::Compare(
target, current_ == mutable_iter_ ? immutable_min_heap_.top()->key() target, current_ == mutable_iter_ ? immutable_min_heap_.top()->key()
: current_->key()) > 0) { : current_->key()) > 0) {
return true; return true;
} }
return false; return false;
@ -1040,11 +1034,11 @@ uint32_t ForwardIterator::FindFileInRange(
uint32_t left, uint32_t right) { uint32_t left, uint32_t right) {
auto cmp = [&](const FileMetaData* f, const Slice& k) -> bool { auto cmp = [&](const FileMetaData* f, const Slice& k) -> bool {
return cfd_->internal_comparator().InternalKeyComparator::Compare( return cfd_->internal_comparator().InternalKeyComparator::Compare(
f->largest.Encode(), k) < 0; f->largest.Encode(), k) < 0;
}; };
const auto &b = files.begin(); const auto& b = files.begin();
return static_cast<uint32_t>(std::lower_bound(b + left, return static_cast<uint32_t>(
b + right, internal_key, cmp) - b); std::lower_bound(b + left, b + right, internal_key, cmp) - b);
} }
void ForwardIterator::DeleteIterator(InternalIterator* iter, bool is_arena) { void ForwardIterator::DeleteIterator(InternalIterator* iter, bool is_arena) {

@ -7,9 +7,9 @@
#include "rocksdb/comparator.h" #include "rocksdb/comparator.h"
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#include <queue>
#include <string> #include <string>
#include <vector> #include <vector>
#include <queue>
#include "memory/arena.h" #include "memory/arena.h"
#include "rocksdb/db.h" #include "rocksdb/db.h"
@ -35,6 +35,7 @@ class MinIterComparator {
bool operator()(InternalIterator* a, InternalIterator* b) { bool operator()(InternalIterator* a, InternalIterator* b) {
return comparator_->Compare(a->key(), b->key()) > 0; return comparator_->Compare(a->key(), b->key()) > 0;
} }
private: private:
const CompareInterface* comparator_; const CompareInterface* comparator_;
}; };
@ -92,8 +93,8 @@ class ForwardIterator : public InternalIterator {
// either done immediately or deferred until this iterator is unpinned by // either done immediately or deferred until this iterator is unpinned by
// PinnedIteratorsManager. // PinnedIteratorsManager.
void SVCleanup(); void SVCleanup();
static void SVCleanup( static void SVCleanup(DBImpl* db, SuperVersion* sv,
DBImpl* db, SuperVersion* sv, bool background_purge_on_iterator_cleanup); bool background_purge_on_iterator_cleanup);
static void DeferredSVCleanup(void* arg); static void DeferredSVCleanup(void* arg);
void RebuildIterators(bool refresh_sv); void RebuildIterators(bool refresh_sv);
@ -107,9 +108,9 @@ class ForwardIterator : public InternalIterator {
void UpdateCurrent(); void UpdateCurrent();
bool NeedToSeekImmutable(const Slice& internal_key); bool NeedToSeekImmutable(const Slice& internal_key);
void DeleteCurrentIter(); void DeleteCurrentIter();
uint32_t FindFileInRange( uint32_t FindFileInRange(const std::vector<FileMetaData*>& files,
const std::vector<FileMetaData*>& files, const Slice& internal_key, const Slice& internal_key, uint32_t left,
uint32_t left, uint32_t right); uint32_t right);
bool IsOverUpperBound(const Slice& internal_key) const; bool IsOverUpperBound(const Slice& internal_key) const;

@ -14,6 +14,7 @@ int main() {
int main() { return 0; } int main() { return 0; }
#else #else
#include <semaphore.h> #include <semaphore.h>
#include <atomic> #include <atomic>
#include <bitset> #include <bitset>
#include <chrono> #include <chrono>
@ -281,8 +282,9 @@ struct StatsThread {
} }
auto now = std::chrono::steady_clock::now(); auto now = std::chrono::steady_clock::now();
double elapsed = double elapsed =
std::chrono::duration_cast<std::chrono::duration<double> >( std::chrono::duration_cast<std::chrono::duration<double> >(now -
now - tlast).count(); tlast)
.count();
uint64_t w = ::stats.written.load(); uint64_t w = ::stats.written.load();
uint64_t r = ::stats.read.load(); uint64_t r = ::stats.read.load();
fprintf(stderr, fprintf(stderr,

@ -228,8 +228,8 @@ Status ImportColumnFamilyJob::GetIngestedFileInfo(
std::unique_ptr<FSRandomAccessFile> sst_file; std::unique_ptr<FSRandomAccessFile> sst_file;
std::unique_ptr<RandomAccessFileReader> sst_file_reader; std::unique_ptr<RandomAccessFileReader> sst_file_reader;
status = fs_->NewRandomAccessFile(external_file, env_options_, status =
&sst_file, nullptr); fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }

@ -353,7 +353,7 @@ class InternalStats {
this->num_output_records += c.num_output_records; this->num_output_records += c.num_output_records;
this->count += c.count; this->count += c.count;
int num_of_reasons = static_cast<int>(CompactionReason::kNumOfReasons); int num_of_reasons = static_cast<int>(CompactionReason::kNumOfReasons);
for (int i = 0; i< num_of_reasons; i++) { for (int i = 0; i < num_of_reasons; i++) {
counts[i] += c.counts[i]; counts[i] += c.counts[i];
} }
} }
@ -648,8 +648,8 @@ class InternalStats {
struct CFStatsSnapshot { struct CFStatsSnapshot {
// ColumnFamily-level stats // ColumnFamily-level stats
CompactionStats comp_stats; CompactionStats comp_stats;
uint64_t ingest_bytes_flush; // Bytes written to L0 (Flush) uint64_t ingest_bytes_flush; // Bytes written to L0 (Flush)
uint64_t stall_count; // Stall count uint64_t stall_count; // Stall count
// Stats from compaction jobs - bytes written, bytes read, duration. // Stats from compaction jobs - bytes written, bytes read, duration.
uint64_t compact_bytes_write; uint64_t compact_bytes_write;
uint64_t compact_bytes_read; uint64_t compact_bytes_read;
@ -691,10 +691,10 @@ class InternalStats {
struct DBStatsSnapshot { struct DBStatsSnapshot {
// DB-level stats // DB-level stats
uint64_t ingest_bytes; // Bytes written by user uint64_t ingest_bytes; // Bytes written by user
uint64_t wal_bytes; // Bytes written to WAL uint64_t wal_bytes; // Bytes written to WAL
uint64_t wal_synced; // Number of times WAL is synced uint64_t wal_synced; // Number of times WAL is synced
uint64_t write_with_wal; // Number of writes that request WAL uint64_t write_with_wal; // Number of writes that request WAL
// These count the number of writes processed by the calling thread or // These count the number of writes processed by the calling thread or
// another thread. // another thread.
uint64_t write_other; uint64_t write_other;
@ -980,13 +980,14 @@ class InternalStats {
return false; return false;
} }
bool GetIntProperty(const DBPropertyInfo& /*property_info*/, uint64_t* /*value*/, bool GetIntProperty(const DBPropertyInfo& /*property_info*/,
DBImpl* /*db*/) const { uint64_t* /*value*/, DBImpl* /*db*/) const {
return false; return false;
} }
bool GetIntPropertyOutOfMutex(const DBPropertyInfo& /*property_info*/, bool GetIntPropertyOutOfMutex(const DBPropertyInfo& /*property_info*/,
Version* /*version*/, uint64_t* /*value*/) const { Version* /*version*/,
uint64_t* /*value*/) const {
return false; return false;
} }
}; };

@ -35,7 +35,7 @@ struct SuperVersionContext {
new_superversion; // if nullptr no new superversion new_superversion; // if nullptr no new superversion
explicit SuperVersionContext(bool create_superversion = false) explicit SuperVersionContext(bool create_superversion = false)
: new_superversion(create_superversion ? new SuperVersion() : nullptr) {} : new_superversion(create_superversion ? new SuperVersion() : nullptr) {}
explicit SuperVersionContext(SuperVersionContext&& other) noexcept explicit SuperVersionContext(SuperVersionContext&& other) noexcept
: superversions_to_free(std::move(other.superversions_to_free)), : superversions_to_free(std::move(other.superversions_to_free)),
@ -54,8 +54,7 @@ struct SuperVersionContext {
inline bool HaveSomethingToDelete() const { inline bool HaveSomethingToDelete() const {
#ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION #ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION
return !superversions_to_free.empty() || return !superversions_to_free.empty() || !write_stall_notifications.empty();
!write_stall_notifications.empty();
#else #else
return !superversions_to_free.empty(); return !superversions_to_free.empty();
#endif #endif
@ -77,7 +76,8 @@ struct SuperVersionContext {
(void)new_cond; (void)new_cond;
(void)name; (void)name;
(void)ioptions; (void)ioptions;
#endif // !defined(ROCKSDB_LITE) && !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION) #endif // !defined(ROCKSDB_LITE) &&
// !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION)
} }
void Clean() { void Clean() {
@ -139,8 +139,7 @@ struct JobContext {
CandidateFileInfo(std::string name, std::string path) CandidateFileInfo(std::string name, std::string path)
: file_name(std::move(name)), file_path(std::move(path)) {} : file_name(std::move(name)), file_path(std::move(path)) {}
bool operator==(const CandidateFileInfo& other) const { bool operator==(const CandidateFileInfo& other) const {
return file_name == other.file_name && return file_name == other.file_name && file_path == other.file_path;
file_path == other.file_path;
} }
}; };

@ -89,7 +89,7 @@ class TestCompactionListener : public EventListener {
public: public:
explicit TestCompactionListener(EventListenerTest* test) : test_(test) {} explicit TestCompactionListener(EventListenerTest* test) : test_(test) {}
void OnCompactionCompleted(DB *db, const CompactionJobInfo& ci) override { void OnCompactionCompleted(DB* db, const CompactionJobInfo& ci) override {
std::lock_guard<std::mutex> lock(mutex_); std::lock_guard<std::mutex> lock(mutex_);
compacted_dbs_.push_back(db); compacted_dbs_.push_back(db);
ASSERT_GT(ci.input_files.size(), 0U); ASSERT_GT(ci.input_files.size(), 0U);
@ -172,9 +172,9 @@ TEST_F(EventListenerTest, OnSingleDBCompactionTest) {
TestCompactionListener* listener = new TestCompactionListener(this); TestCompactionListener* listener = new TestCompactionListener(this);
options.listeners.emplace_back(listener); options.listeners.emplace_back(listener);
std::vector<std::string> cf_names = { std::vector<std::string> cf_names = {"pikachu", "ilya", "muromec",
"pikachu", "ilya", "muromec", "dobrynia", "dobrynia", "nikitich", "alyosha",
"nikitich", "alyosha", "popovich"}; "popovich"};
CreateAndReopenWithCF(cf_names, options); CreateAndReopenWithCF(cf_names, options);
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p'))); ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
@ -214,8 +214,7 @@ class TestFlushListener : public EventListener {
virtual ~TestFlushListener() { virtual ~TestFlushListener() {
prev_fc_info_.status.PermitUncheckedError(); // Ignore the status prev_fc_info_.status.PermitUncheckedError(); // Ignore the status
} }
void OnTableFileCreated( void OnTableFileCreated(const TableFileCreationInfo& info) override {
const TableFileCreationInfo& info) override {
// remember the info for later checking the FlushJobInfo. // remember the info for later checking the FlushJobInfo.
prev_fc_info_ = info; prev_fc_info_ = info;
ASSERT_GT(info.db_name.size(), 0U); ASSERT_GT(info.db_name.size(), 0U);
@ -250,8 +249,7 @@ class TestFlushListener : public EventListener {
#endif // ROCKSDB_USING_THREAD_STATUS #endif // ROCKSDB_USING_THREAD_STATUS
} }
void OnFlushCompleted( void OnFlushCompleted(DB* db, const FlushJobInfo& info) override {
DB* db, const FlushJobInfo& info) override {
flushed_dbs_.push_back(db); flushed_dbs_.push_back(db);
flushed_column_family_names_.push_back(info.cf_name); flushed_column_family_names_.push_back(info.cf_name);
if (info.triggered_writes_slowdown) { if (info.triggered_writes_slowdown) {
@ -317,9 +315,9 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) {
#endif // ROCKSDB_USING_THREAD_STATUS #endif // ROCKSDB_USING_THREAD_STATUS
TestFlushListener* listener = new TestFlushListener(options.env, this); TestFlushListener* listener = new TestFlushListener(options.env, this);
options.listeners.emplace_back(listener); options.listeners.emplace_back(listener);
std::vector<std::string> cf_names = { std::vector<std::string> cf_names = {"pikachu", "ilya", "muromec",
"pikachu", "ilya", "muromec", "dobrynia", "dobrynia", "nikitich", "alyosha",
"nikitich", "alyosha", "popovich"}; "popovich"};
options.table_properties_collector_factories.push_back( options.table_properties_collector_factories.push_back(
std::make_shared<TestPropertiesCollectorFactory>()); std::make_shared<TestPropertiesCollectorFactory>());
CreateAndReopenWithCF(cf_names, options); CreateAndReopenWithCF(cf_names, options);
@ -421,9 +419,9 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
listeners.emplace_back(new TestFlushListener(options.env, this)); listeners.emplace_back(new TestFlushListener(options.env, this));
} }
std::vector<std::string> cf_names = { std::vector<std::string> cf_names = {"pikachu", "ilya", "muromec",
"pikachu", "ilya", "muromec", "dobrynia", "dobrynia", "nikitich", "alyosha",
"nikitich", "alyosha", "popovich"}; "popovich"};
options.create_if_missing = true; options.create_if_missing = true;
for (int i = 0; i < kNumListeners; ++i) { for (int i = 0; i < kNumListeners; ++i) {
@ -433,7 +431,7 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
ColumnFamilyOptions cf_opts(options); ColumnFamilyOptions cf_opts(options);
std::vector<DB*> dbs; std::vector<DB*> dbs;
std::vector<std::vector<ColumnFamilyHandle *>> vec_handles; std::vector<std::vector<ColumnFamilyHandle*>> vec_handles;
for (int d = 0; d < kNumDBs; ++d) { for (int d = 0; d < kNumDBs; ++d) {
ASSERT_OK(DestroyDB(dbname_ + std::to_string(d), options)); ASSERT_OK(DestroyDB(dbname_ + std::to_string(d), options));
@ -452,8 +450,8 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
for (int d = 0; d < kNumDBs; ++d) { for (int d = 0; d < kNumDBs; ++d) {
for (size_t c = 0; c < cf_names.size(); ++c) { for (size_t c = 0; c < cf_names.size(); ++c) {
ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c], ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c], cf_names[c],
cf_names[c], cf_names[c])); cf_names[c]));
} }
} }
@ -483,7 +481,6 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
} }
} }
for (auto handles : vec_handles) { for (auto handles : vec_handles) {
for (auto h : handles) { for (auto h : handles) {
delete h; delete h;
@ -887,16 +884,17 @@ TEST_F(EventListenerTest, TableFileCreationListenersTest) {
} }
class MemTableSealedListener : public EventListener { class MemTableSealedListener : public EventListener {
private: private:
SequenceNumber latest_seq_number_; SequenceNumber latest_seq_number_;
public:
public:
MemTableSealedListener() {} MemTableSealedListener() {}
void OnMemTableSealed(const MemTableInfo& info) override { void OnMemTableSealed(const MemTableInfo& info) override {
latest_seq_number_ = info.first_seqno; latest_seq_number_ = info.first_seqno;
} }
void OnFlushCompleted(DB* /*db*/, void OnFlushCompleted(DB* /*db*/,
const FlushJobInfo& flush_job_info) override { const FlushJobInfo& flush_job_info) override {
ASSERT_LE(flush_job_info.smallest_seqno, latest_seq_number_); ASSERT_LE(flush_job_info.smallest_seqno, latest_seq_number_);
} }
}; };
@ -911,8 +909,8 @@ TEST_F(EventListenerTest, MemTableSealedListenerTest) {
for (unsigned int i = 0; i < 10; i++) { for (unsigned int i = 0; i < 10; i++) {
std::string tag = std::to_string(i); std::string tag = std::to_string(i);
ASSERT_OK(Put("foo"+tag, "aaa")); ASSERT_OK(Put("foo" + tag, "aaa"));
ASSERT_OK(Put("bar"+tag, "bbb")); ASSERT_OK(Put("bar" + tag, "bbb"));
ASSERT_OK(Flush()); ASSERT_OK(Flush());
} }

@ -21,8 +21,7 @@
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
namespace log { namespace log {
Reader::Reporter::~Reporter() { Reader::Reporter::~Reporter() {}
}
Reader::Reader(std::shared_ptr<Logger> info_log, Reader::Reader(std::shared_ptr<Logger> info_log,
std::unique_ptr<SequentialFileReader>&& _file, std::unique_ptr<SequentialFileReader>&& _file,
@ -241,9 +240,8 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
FALLTHROUGH_INTENDED; FALLTHROUGH_INTENDED;
case kBadRecordChecksum: case kBadRecordChecksum:
if (recycled_ && if (recycled_ && wal_recovery_mode ==
wal_recovery_mode == WALRecoveryMode::kTolerateCorruptedTailRecords) {
WALRecoveryMode::kTolerateCorruptedTailRecords) {
scratch->clear(); scratch->clear();
return false; return false;
} }
@ -297,9 +295,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
return false; return false;
} }
uint64_t Reader::LastRecordOffset() { uint64_t Reader::LastRecordOffset() { return last_record_offset_; }
return last_record_offset_;
}
uint64_t Reader::LastRecordEnd() { uint64_t Reader::LastRecordEnd() {
return end_of_buffer_offset_ - buffer_.size(); return end_of_buffer_offset_ - buffer_.size();
@ -361,11 +357,11 @@ void Reader::UnmarkEOFInternal() {
if (read_buffer.data() != backing_store_ + eof_offset_) { if (read_buffer.data() != backing_store_ + eof_offset_) {
// Read did not write to backing_store_ // Read did not write to backing_store_
memmove(backing_store_ + eof_offset_, read_buffer.data(), memmove(backing_store_ + eof_offset_, read_buffer.data(),
read_buffer.size()); read_buffer.size());
} }
buffer_ = Slice(backing_store_ + consumed_bytes, buffer_ = Slice(backing_store_ + consumed_bytes,
eof_offset_ + added - consumed_bytes); eof_offset_ + added - consumed_bytes);
if (added < remaining) { if (added < remaining) {
eof_ = true; eof_ = true;
@ -385,7 +381,7 @@ void Reader::ReportDrop(size_t bytes, const Status& reason) {
} }
} }
bool Reader::ReadMore(size_t* drop_size, int *error) { bool Reader::ReadMore(size_t* drop_size, int* error) {
if (!eof_ && !read_error_) { if (!eof_ && !read_error_) {
// Last read was a full read, so this is a trailer to skip // Last read was a full read, so this is a trailer to skip
buffer_.clear(); buffer_.clear();

@ -85,9 +85,7 @@ class Reader {
uint64_t LastRecordEnd(); uint64_t LastRecordEnd();
// returns true if the reader has encountered an eof condition. // returns true if the reader has encountered an eof condition.
bool IsEOF() { bool IsEOF() { return eof_; }
return eof_;
}
// returns true if the reader has encountered read error. // returns true if the reader has encountered read error.
bool hasReadError() const { return read_error_; } bool hasReadError() const { return read_error_; }
@ -122,8 +120,8 @@ class Reader {
// Internal state variables used for reading records // Internal state variables used for reading records
Slice buffer_; Slice buffer_;
bool eof_; // Last Read() indicated EOF by returning < kBlockSize bool eof_; // Last Read() indicated EOF by returning < kBlockSize
bool read_error_; // Error occurred while reading from file bool read_error_; // Error occurred while reading from file
// Offset of the file position indicator within the last block when an // Offset of the file position indicator within the last block when an
// EOF was detected. // EOF was detected.
@ -182,7 +180,7 @@ class Reader {
uint64_t* fragment_checksum = nullptr); uint64_t* fragment_checksum = nullptr);
// Read some more // Read some more
bool ReadMore(size_t* drop_size, int *error); bool ReadMore(size_t* drop_size, int* error);
void UnmarkEOFInternal(); void UnmarkEOFInternal();

@ -128,7 +128,7 @@ class LogTest
size_t dropped_bytes_; size_t dropped_bytes_;
std::string message_; std::string message_;
ReportCollector() : dropped_bytes_(0) { } ReportCollector() : dropped_bytes_(0) {}
void Corruption(size_t bytes, const Status& status) override { void Corruption(size_t bytes, const Status& status) override {
dropped_bytes_ += bytes; dropped_bytes_ += bytes;
message_.append(status.ToString()); message_.append(status.ToString());
@ -185,9 +185,7 @@ class LogTest
ASSERT_OK(writer_->AddRecord(Slice(msg))); ASSERT_OK(writer_->AddRecord(Slice(msg)));
} }
size_t WrittenBytes() const { size_t WrittenBytes() const { return dest_contents().size(); }
return dest_contents().size();
}
std::string Read(const WALRecoveryMode wal_recovery_mode = std::string Read(const WALRecoveryMode wal_recovery_mode =
WALRecoveryMode::kTolerateCorruptedTailRecords) { WALRecoveryMode::kTolerateCorruptedTailRecords) {
@ -235,13 +233,9 @@ class LogTest
source_->force_error_position_ = position; source_->force_error_position_ = position;
} }
size_t DroppedBytes() const { size_t DroppedBytes() const { return report_.dropped_bytes_; }
return report_.dropped_bytes_;
}
std::string ReportMessage() const { std::string ReportMessage() const { return report_.message_; }
return report_.message_;
}
void ForceEOF(size_t position = 0) { void ForceEOF(size_t position = 0) {
source_->force_eof_ = true; source_->force_eof_ = true;
@ -389,7 +383,7 @@ TEST_P(LogTest, BadRecordType) {
TEST_P(LogTest, TruncatedTrailingRecordIsIgnored) { TEST_P(LogTest, TruncatedTrailingRecordIsIgnored) {
Write("foo"); Write("foo");
ShrinkSize(4); // Drop all payload as well as a header byte ShrinkSize(4); // Drop all payload as well as a header byte
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
// Truncated last record is ignored, not treated as an error // Truncated last record is ignored, not treated as an error
ASSERT_EQ(0U, DroppedBytes()); ASSERT_EQ(0U, DroppedBytes());
@ -581,7 +575,7 @@ TEST_P(LogTest, ErrorJoinsRecords) {
Write("correct"); Write("correct");
// Wipe the middle block // Wipe the middle block
for (unsigned int offset = kBlockSize; offset < 2*kBlockSize; offset++) { for (unsigned int offset = kBlockSize; offset < 2 * kBlockSize; offset++) {
SetByte(offset, 'x'); SetByte(offset, 'x');
} }

@ -100,7 +100,7 @@ class Writer {
private: private:
std::unique_ptr<WritableFileWriter> dest_; std::unique_ptr<WritableFileWriter> dest_;
size_t block_offset_; // Current offset in block size_t block_offset_; // Current offset in block
uint64_t log_number_; uint64_t log_number_;
bool recycle_log_files_; bool recycle_log_files_;

@ -58,6 +58,5 @@ class LogsWithPrepTracker {
// both logs_with_prep_ and prepared_section_completed_. // both logs_with_prep_ and prepared_section_completed_.
std::unordered_map<uint64_t, uint64_t> prepared_section_completed_; std::unordered_map<uint64_t, uint64_t> prepared_section_completed_;
std::mutex prepared_section_completed_mutex_; std::mutex prepared_section_completed_mutex_;
}; };
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -10,6 +10,7 @@
#pragma once #pragma once
#include <string> #include <string>
#include <utility> #include <utility>
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "rocksdb/types.h" #include "rocksdb/types.h"

@ -10,9 +10,10 @@
#include "db/malloc_stats.h" #include "db/malloc_stats.h"
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#include <memory>
#include <string.h> #include <string.h>
#include <memory>
#include "port/jemalloc_helper.h" #include "port/jemalloc_helper.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -42,9 +42,7 @@ std::string Key1(int i) {
return buf; return buf;
} }
std::string Key2(int i) { std::string Key2(int i) { return Key1(i) + "_xxx"; }
return Key1(i) + "_xxx";
}
class ManualCompactionTest : public testing::Test { class ManualCompactionTest : public testing::Test {
public: public:
@ -102,10 +100,10 @@ TEST_F(ManualCompactionTest, CompactTouchesAllKeys) {
for (int iter = 0; iter < 2; ++iter) { for (int iter = 0; iter < 2; ++iter) {
DB* db; DB* db;
Options options; Options options;
if (iter == 0) { // level compaction if (iter == 0) { // level compaction
options.num_levels = 3; options.num_levels = 3;
options.compaction_style = CompactionStyle::kCompactionStyleLevel; options.compaction_style = CompactionStyle::kCompactionStyleLevel;
} else { // universal compaction } else { // universal compaction
options.compaction_style = CompactionStyle::kCompactionStyleUniversal; options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
} }
options.create_if_missing = true; options.create_if_missing = true;

@ -330,9 +330,8 @@ int MemTable::KeyComparator::operator()(const char* prefix_len_key1,
return comparator.CompareKeySeq(k1, k2); return comparator.CompareKeySeq(k1, k2);
} }
int MemTable::KeyComparator::operator()(const char* prefix_len_key, int MemTable::KeyComparator::operator()(
const KeyComparator::DecodedType& key) const char* prefix_len_key, const KeyComparator::DecodedType& key) const {
const {
// Internal keys are encoded as length-prefixed strings. // Internal keys are encoded as length-prefixed strings.
Slice a = GetLengthPrefixedSlice(prefix_len_key); Slice a = GetLengthPrefixedSlice(prefix_len_key);
return comparator.CompareKeySeq(a, key); return comparator.CompareKeySeq(a, key);
@ -914,7 +913,7 @@ struct Saver {
return true; return true;
} }
}; };
} // namespace } // anonymous namespace
static bool SaveValue(void* arg, const char* entry) { static bool SaveValue(void* arg, const char* entry) {
TEST_SYNC_POINT_CALLBACK("Memtable::SaveValue:Begin:entry", &entry); TEST_SYNC_POINT_CALLBACK("Memtable::SaveValue:Begin:entry", &entry);

@ -88,7 +88,7 @@ class MemTable {
public: public:
struct KeyComparator : public MemTableRep::KeyComparator { struct KeyComparator : public MemTableRep::KeyComparator {
const InternalKeyComparator comparator; const InternalKeyComparator comparator;
explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { } explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {}
virtual int operator()(const char* prefix_len_key1, virtual int operator()(const char* prefix_len_key1,
const char* prefix_len_key2) const override; const char* prefix_len_key2) const override;
virtual int operator()(const char* prefix_len_key, virtual int operator()(const char* prefix_len_key,
@ -448,9 +448,7 @@ class MemTable {
// persisted. // persisted.
// REQUIRES: external synchronization to prevent simultaneous // REQUIRES: external synchronization to prevent simultaneous
// operations on the same MemTable. // operations on the same MemTable.
void MarkFlushed() { void MarkFlushed() { table_->MarkFlushed(); }
table_->MarkFlushed();
}
// return true if the current MemTableRep supports merge operator. // return true if the current MemTableRep supports merge operator.
bool IsMergeOperatorSupported() const { bool IsMergeOperatorSupported() const {
@ -562,8 +560,8 @@ class MemTable {
std::atomic<size_t> write_buffer_size_; std::atomic<size_t> write_buffer_size_;
// These are used to manage memtable flushes to storage // These are used to manage memtable flushes to storage
bool flush_in_progress_; // started the flush bool flush_in_progress_; // started the flush
bool flush_completed_; // finished the flush bool flush_completed_; // finished the flush
uint64_t file_number_; // filled up after flush is complete uint64_t file_number_; // filled up after flush is complete
// The updates to be applied to the transaction log when this // The updates to be applied to the transaction log when this

@ -4,9 +4,11 @@
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#include "db/memtable_list.h" #include "db/memtable_list.h"
#include <algorithm> #include <algorithm>
#include <string> #include <string>
#include <vector> #include <vector>
#include "db/merge_context.h" #include "db/merge_context.h"
#include "db/version_set.h" #include "db/version_set.h"
#include "db/write_controller.h" #include "db/write_controller.h"

@ -8,6 +8,7 @@
#include <memory> #include <memory>
#include <string> #include <string>
#include <vector> #include <vector>
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -167,7 +167,7 @@ class MergeHelper {
const CompactionFilter* compaction_filter_; const CompactionFilter* compaction_filter_;
const std::atomic<bool>* shutting_down_; const std::atomic<bool>* shutting_down_;
Logger* logger_; Logger* logger_;
bool assert_valid_internal_key_; // enforce no internal key corruption? bool assert_valid_internal_key_; // enforce no internal key corruption?
bool allow_single_operand_; bool allow_single_operand_;
SequenceNumber latest_snapshot_; SequenceNumber latest_snapshot_;
const SnapshotChecker* const snapshot_checker_; const SnapshotChecker* const snapshot_checker_;

@ -74,12 +74,11 @@ bool AssociativeMergeOperator::FullMergeV2(
// Call the user defined simple merge on the operands; // Call the user defined simple merge on the operands;
// NOTE: It is assumed that the client's merge-operator will handle any errors. // NOTE: It is assumed that the client's merge-operator will handle any errors.
bool AssociativeMergeOperator::PartialMerge( bool AssociativeMergeOperator::PartialMerge(const Slice& key,
const Slice& key, const Slice& left_operand,
const Slice& left_operand, const Slice& right_operand,
const Slice& right_operand, std::string* new_value,
std::string* new_value, Logger* logger) const {
Logger* logger) const {
return Merge(key, &left_operand, right_operand, new_value, logger); return Merge(key, &left_operand, right_operand, new_value, logger);
} }

@ -10,10 +10,12 @@
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#include <stdlib.h> #include <stdlib.h>
#include <algorithm> #include <algorithm>
#include <map> #include <map>
#include <string> #include <string>
#include <vector> #include <vector>
#include "db/db_impl/db_impl.h" #include "db/db_impl/db_impl.h"
#include "db/db_test_util.h" #include "db/db_test_util.h"
#include "db/version_set.h" #include "db/version_set.h"
@ -28,7 +30,6 @@
#include "test_util/testutil.h" #include "test_util/testutil.h"
#include "util/string_util.h" #include "util/string_util.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
class ObsoleteFilesTest : public DBTestBase { class ObsoleteFilesTest : public DBTestBase {
@ -40,7 +41,7 @@ class ObsoleteFilesTest : public DBTestBase {
void AddKeys(int numkeys, int startkey) { void AddKeys(int numkeys, int startkey) {
WriteOptions options; WriteOptions options;
options.sync = false; options.sync = false;
for (int i = startkey; i < (numkeys + startkey) ; i++) { for (int i = startkey; i < (numkeys + startkey); i++) {
std::string temp = std::to_string(i); std::string temp = std::to_string(i);
Slice key(temp); Slice key(temp);
Slice value(temp); Slice value(temp);
@ -117,7 +118,7 @@ TEST_F(ObsoleteFilesTest, RaceForObsoleteFileDeletion) {
"ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"}, "ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"},
{"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles", {"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles",
"ObsoleteFilesTest::RaceForObsoleteFileDeletion:2"}, "ObsoleteFilesTest::RaceForObsoleteFileDeletion:2"},
}); });
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
"DBImpl::DeleteObsoleteFileImpl:AfterDeletion", [&](void* arg) { "DBImpl::DeleteObsoleteFileImpl:AfterDeletion", [&](void* arg) {
Status* p_status = reinterpret_cast<Status*>(arg); Status* p_status = reinterpret_cast<Status*>(arg);

@ -59,7 +59,7 @@ void VerifyOptionsFileName(
} }
} }
} }
} // namespace } // anonymous namespace
TEST_F(OptionsFileTest, NumberOfOptionsFiles) { TEST_F(OptionsFileTest, NumberOfOptionsFiles) {
const int kReopenCount = 20; const int kReopenCount = 20;

@ -39,31 +39,31 @@ const std::string kDbName =
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
std::shared_ptr<DB> OpenDb(bool read_only = false) { std::shared_ptr<DB> OpenDb(bool read_only = false) {
DB* db; DB* db;
Options options; Options options;
options.create_if_missing = true; options.create_if_missing = true;
options.max_open_files = -1; options.max_open_files = -1;
options.write_buffer_size = FLAGS_write_buffer_size; options.write_buffer_size = FLAGS_write_buffer_size;
options.max_write_buffer_number = FLAGS_max_write_buffer_number; options.max_write_buffer_number = FLAGS_max_write_buffer_number;
options.min_write_buffer_number_to_merge = options.min_write_buffer_number_to_merge =
FLAGS_min_write_buffer_number_to_merge; FLAGS_min_write_buffer_number_to_merge;
if (FLAGS_use_set_based_memetable) { if (FLAGS_use_set_based_memetable) {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
options.prefix_extractor.reset( options.prefix_extractor.reset(
ROCKSDB_NAMESPACE::NewFixedPrefixTransform(0)); ROCKSDB_NAMESPACE::NewFixedPrefixTransform(0));
options.memtable_factory.reset(NewHashSkipListRepFactory()); options.memtable_factory.reset(NewHashSkipListRepFactory());
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
} }
Status s; Status s;
if (!read_only) { if (!read_only) {
s = DB::Open(options, kDbName, &db); s = DB::Open(options, kDbName, &db);
} else { } else {
s = DB::OpenForReadOnly(options, kDbName, &db); s = DB::OpenForReadOnly(options, kDbName, &db);
} }
EXPECT_OK(s); EXPECT_OK(s);
return std::shared_ptr<DB>(db); return std::shared_ptr<DB>(db);
} }
class PerfContextTest : public testing::Test {}; class PerfContextTest : public testing::Test {};
@ -81,7 +81,7 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
ASSERT_OK(db->Put(write_options, key, value)); ASSERT_OK(db->Put(write_options, key, value));
} }
for (int i = 0; i < FLAGS_total_keys -1 ; ++i) { for (int i = 0; i < FLAGS_total_keys - 1; ++i) {
std::string key = "k" + std::to_string(i); std::string key = "k" + std::to_string(i);
ASSERT_OK(db->Delete(write_options, key)); ASSERT_OK(db->Delete(write_options, key));
} }
@ -103,8 +103,9 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
} }
if (FLAGS_verbose) { if (FLAGS_verbose) {
std::cout << "Get user key comparison: \n" << hist_get.ToString() std::cout << "Get user key comparison: \n"
<< "Get time: \n" << hist_get_time.ToString(); << hist_get.ToString() << "Get time: \n"
<< hist_get_time.ToString();
} }
{ {
@ -139,7 +140,8 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
hist_seek.Add(get_perf_context()->user_key_comparison_count); hist_seek.Add(get_perf_context()->user_key_comparison_count);
if (FLAGS_verbose) { if (FLAGS_verbose) {
std::cout << "seek cmp: " << get_perf_context()->user_key_comparison_count std::cout << "seek cmp: " << get_perf_context()->user_key_comparison_count
<< " ikey skipped " << get_perf_context()->internal_key_skipped_count << " ikey skipped "
<< get_perf_context()->internal_key_skipped_count
<< " idelete skipped " << " idelete skipped "
<< get_perf_context()->internal_delete_skipped_count << get_perf_context()->internal_delete_skipped_count
<< " elapsed: " << elapsed_nanos << "ns\n"; << " elapsed: " << elapsed_nanos << "ns\n";
@ -322,7 +324,8 @@ void ProfileQueries(bool enabled_time = false) {
hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time); hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time); hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
hist_mget_files.Add(get_perf_context()->get_from_output_files_time); hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
hist_mget_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count); hist_mget_num_memtable_checked.Add(
get_perf_context()->get_from_memtable_count);
hist_mget_post_process.Add(get_perf_context()->get_post_process_time); hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
hist_mget.Add(get_perf_context()->user_key_comparison_count); hist_mget.Add(get_perf_context()->user_key_comparison_count);
} }
@ -337,12 +340,14 @@ void ProfileQueries(bool enabled_time = false) {
<< hist_write_wal_time.ToString() << "\n" << hist_write_wal_time.ToString() << "\n"
<< " Writing Mem Table time: \n" << " Writing Mem Table time: \n"
<< hist_write_memtable_time.ToString() << "\n" << hist_write_memtable_time.ToString() << "\n"
<< " Write Delay: \n" << hist_write_delay_time.ToString() << "\n" << " Write Delay: \n"
<< hist_write_delay_time.ToString() << "\n"
<< " Waiting for Batch time: \n" << " Waiting for Batch time: \n"
<< hist_write_thread_wait_nanos.ToString() << "\n" << hist_write_thread_wait_nanos.ToString() << "\n"
<< " Scheduling Flushes and Compactions Time: \n" << " Scheduling Flushes and Compactions Time: \n"
<< hist_write_scheduling_time.ToString() << "\n" << hist_write_scheduling_time.ToString() << "\n"
<< " Total DB mutex nanos: \n" << total_db_mutex_nanos << "\n"; << " Total DB mutex nanos: \n"
<< total_db_mutex_nanos << "\n";
std::cout << "Get(): Time to get snapshot: \n" std::cout << "Get(): Time to get snapshot: \n"
<< hist_get_snapshot.ToString() << hist_get_snapshot.ToString()
@ -352,8 +357,8 @@ void ProfileQueries(bool enabled_time = false) {
<< hist_get_files.ToString() << "\n" << hist_get_files.ToString() << "\n"
<< " Number of memtables checked: \n" << " Number of memtables checked: \n"
<< hist_num_memtable_checked.ToString() << "\n" << hist_num_memtable_checked.ToString() << "\n"
<< " Time to post process: \n" << hist_get_post_process.ToString() << " Time to post process: \n"
<< "\n"; << hist_get_post_process.ToString() << "\n";
std::cout << "MultiGet(): Time to get snapshot: \n" std::cout << "MultiGet(): Time to get snapshot: \n"
<< hist_mget_snapshot.ToString() << hist_mget_snapshot.ToString()
@ -440,7 +445,8 @@ void ProfileQueries(bool enabled_time = false) {
hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time); hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time); hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
hist_mget_files.Add(get_perf_context()->get_from_output_files_time); hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
hist_mget_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count); hist_mget_num_memtable_checked.Add(
get_perf_context()->get_from_memtable_count);
hist_mget_post_process.Add(get_perf_context()->get_post_process_time); hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
hist_mget.Add(get_perf_context()->user_key_comparison_count); hist_mget.Add(get_perf_context()->user_key_comparison_count);
} }
@ -459,8 +465,8 @@ void ProfileQueries(bool enabled_time = false) {
<< hist_get_files.ToString() << "\n" << hist_get_files.ToString() << "\n"
<< " Number of memtables checked: \n" << " Number of memtables checked: \n"
<< hist_num_memtable_checked.ToString() << "\n" << hist_num_memtable_checked.ToString() << "\n"
<< " Time to post process: \n" << hist_get_post_process.ToString() << " Time to post process: \n"
<< "\n"; << hist_get_post_process.ToString() << "\n";
std::cout << "ReadOnly MultiGet(): Time to get snapshot: \n" std::cout << "ReadOnly MultiGet(): Time to get snapshot: \n"
<< hist_mget_snapshot.ToString() << hist_mget_snapshot.ToString()
@ -556,7 +562,8 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
} }
if (FLAGS_verbose) { if (FLAGS_verbose) {
std::cout << "Put time:\n" << hist_put_time.ToString() << "WAL time:\n" std::cout << "Put time:\n"
<< hist_put_time.ToString() << "WAL time:\n"
<< hist_wal_time.ToString() << "time diff:\n" << hist_wal_time.ToString() << "time diff:\n"
<< hist_time_diff.ToString(); << hist_time_diff.ToString();
} }
@ -584,7 +591,8 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
} }
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
if (FLAGS_verbose) { if (FLAGS_verbose) {
std::cout << "Seek:\n" << hist_seek.ToString() << "Next:\n" std::cout << "Seek:\n"
<< hist_seek.ToString() << "Next:\n"
<< hist_next.ToString(); << hist_next.ToString();
} }
} }
@ -614,7 +622,7 @@ TEST_F(PerfContextTest, DBMutexLockCounter) {
SystemClock::Default()->SleepForMicroseconds(100); SystemClock::Default()->SleepForMicroseconds(100);
mutex.Unlock(); mutex.Unlock();
child_thread.join(); child_thread.join();
} }
} }
} }
@ -806,14 +814,18 @@ TEST_F(PerfContextTest, PerfContextByLevelGetSet) {
.bloom_filter_full_positive); .bloom_filter_full_positive);
ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[2] ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[2]
.bloom_filter_full_true_positive); .bloom_filter_full_true_positive);
ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[0] ASSERT_EQ(
.block_cache_hit_count); 1,
ASSERT_EQ(5, (*(get_perf_context()->level_to_perf_context))[2] (*(get_perf_context()->level_to_perf_context))[0].block_cache_hit_count);
.block_cache_hit_count); ASSERT_EQ(
ASSERT_EQ(2, (*(get_perf_context()->level_to_perf_context))[3] 5,
.block_cache_miss_count); (*(get_perf_context()->level_to_perf_context))[2].block_cache_hit_count);
ASSERT_EQ(4, (*(get_perf_context()->level_to_perf_context))[1] ASSERT_EQ(
.block_cache_miss_count); 2,
(*(get_perf_context()->level_to_perf_context))[3].block_cache_miss_count);
ASSERT_EQ(
4,
(*(get_perf_context()->level_to_perf_context))[1].block_cache_miss_count);
std::string zero_excluded = get_perf_context()->ToString(true); std::string zero_excluded = get_perf_context()->ToString(true);
ASSERT_NE(std::string::npos, ASSERT_NE(std::string::npos,
zero_excluded.find("bloom_filter_useful = 1@level5, 2@level7")); zero_excluded.find("bloom_filter_useful = 1@level5, 2@level7"));

@ -38,7 +38,6 @@
#include "util/string_util.h" #include "util/string_util.h"
#include "utilities/merge_operators.h" #include "utilities/merge_operators.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
class PlainTableKeyDecoderTest : public testing::Test {}; class PlainTableKeyDecoderTest : public testing::Test {};
@ -148,9 +147,7 @@ class PlainTableDBTest : public testing::Test,
DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); } DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
void Reopen(Options* options = nullptr) { void Reopen(Options* options = nullptr) { ASSERT_OK(TryReopen(options)); }
ASSERT_OK(TryReopen(options));
}
void Close() { void Close() {
delete db_; delete db_;
@ -160,7 +157,7 @@ class PlainTableDBTest : public testing::Test,
bool mmap_mode() const { return mmap_mode_; } bool mmap_mode() const { return mmap_mode_; }
void DestroyAndReopen(Options* options = nullptr) { void DestroyAndReopen(Options* options = nullptr) {
//Destroy using last options // Destroy using last options
Destroy(&last_options_); Destroy(&last_options_);
ASSERT_OK(TryReopen(options)); ASSERT_OK(TryReopen(options));
} }
@ -200,9 +197,7 @@ class PlainTableDBTest : public testing::Test,
return db_->Put(WriteOptions(), k, v); return db_->Put(WriteOptions(), k, v);
} }
Status Delete(const std::string& k) { Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); }
return db_->Delete(WriteOptions(), k);
}
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
ReadOptions options; ReadOptions options;
@ -217,7 +212,6 @@ class PlainTableDBTest : public testing::Test,
return result; return result;
} }
int NumTableFilesAtLevel(int level) { int NumTableFilesAtLevel(int level) {
std::string property; std::string property;
EXPECT_TRUE(db_->GetProperty( EXPECT_TRUE(db_->GetProperty(
@ -448,99 +442,100 @@ TEST_P(PlainTableDBTest, Flush) {
for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024; for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
huge_page_tlb_size += 2 * 1024 * 1024) { huge_page_tlb_size += 2 * 1024 * 1024) {
for (EncodingType encoding_type : {kPlain, kPrefix}) { for (EncodingType encoding_type : {kPlain, kPrefix}) {
for (int bloom = -1; bloom <= 117; bloom += 117) { for (int bloom = -1; bloom <= 117; bloom += 117) {
const int bloom_bits = std::max(bloom, 0); const int bloom_bits = std::max(bloom, 0);
const bool full_scan_mode = bloom < 0; const bool full_scan_mode = bloom < 0;
for (int total_order = 0; total_order <= 1; total_order++) { for (int total_order = 0; total_order <= 1; total_order++) {
for (int store_index_in_file = 0; store_index_in_file <= 1; for (int store_index_in_file = 0; store_index_in_file <= 1;
++store_index_in_file) { ++store_index_in_file) {
Options options = CurrentOptions(); Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
// Set only one bucket to force bucket conflict. // Set only one bucket to force bucket conflict.
// Test index interval for the same prefix to be 1, 2 and 4 // Test index interval for the same prefix to be 1, 2 and 4
if (total_order) { if (total_order) {
options.prefix_extractor.reset(); options.prefix_extractor.reset();
PlainTableOptions plain_table_options; PlainTableOptions plain_table_options;
plain_table_options.user_key_len = 0; plain_table_options.user_key_len = 0;
plain_table_options.bloom_bits_per_key = bloom_bits; plain_table_options.bloom_bits_per_key = bloom_bits;
plain_table_options.hash_table_ratio = 0; plain_table_options.hash_table_ratio = 0;
plain_table_options.index_sparseness = 2; plain_table_options.index_sparseness = 2;
plain_table_options.huge_page_tlb_size = huge_page_tlb_size; plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
plain_table_options.encoding_type = encoding_type; plain_table_options.encoding_type = encoding_type;
plain_table_options.full_scan_mode = full_scan_mode; plain_table_options.full_scan_mode = full_scan_mode;
plain_table_options.store_index_in_file = store_index_in_file; plain_table_options.store_index_in_file = store_index_in_file;
options.table_factory.reset( options.table_factory.reset(
NewPlainTableFactory(plain_table_options)); NewPlainTableFactory(plain_table_options));
} else { } else {
PlainTableOptions plain_table_options; PlainTableOptions plain_table_options;
plain_table_options.user_key_len = 0; plain_table_options.user_key_len = 0;
plain_table_options.bloom_bits_per_key = bloom_bits; plain_table_options.bloom_bits_per_key = bloom_bits;
plain_table_options.hash_table_ratio = 0.75; plain_table_options.hash_table_ratio = 0.75;
plain_table_options.index_sparseness = 16; plain_table_options.index_sparseness = 16;
plain_table_options.huge_page_tlb_size = huge_page_tlb_size; plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
plain_table_options.encoding_type = encoding_type; plain_table_options.encoding_type = encoding_type;
plain_table_options.full_scan_mode = full_scan_mode; plain_table_options.full_scan_mode = full_scan_mode;
plain_table_options.store_index_in_file = store_index_in_file; plain_table_options.store_index_in_file = store_index_in_file;
options.table_factory.reset( options.table_factory.reset(
NewPlainTableFactory(plain_table_options)); NewPlainTableFactory(plain_table_options));
} }
DestroyAndReopen(&options); DestroyAndReopen(&options);
uint64_t int_num; uint64_t int_num;
ASSERT_TRUE(dbfull()->GetIntProperty( ASSERT_TRUE(dbfull()->GetIntProperty(
"rocksdb.estimate-table-readers-mem", &int_num)); "rocksdb.estimate-table-readers-mem", &int_num));
ASSERT_EQ(int_num, 0U); ASSERT_EQ(int_num, 0U);
ASSERT_OK(Put("1000000000000foo", "v1")); ASSERT_OK(Put("1000000000000foo", "v1"));
ASSERT_OK(Put("0000000000000bar", "v2")); ASSERT_OK(Put("0000000000000bar", "v2"));
ASSERT_OK(Put("1000000000000foo", "v3")); ASSERT_OK(Put("1000000000000foo", "v3"));
ASSERT_OK(dbfull()->TEST_FlushMemTable()); ASSERT_OK(dbfull()->TEST_FlushMemTable());
ASSERT_TRUE(dbfull()->GetIntProperty( ASSERT_TRUE(dbfull()->GetIntProperty(
"rocksdb.estimate-table-readers-mem", &int_num)); "rocksdb.estimate-table-readers-mem", &int_num));
ASSERT_GT(int_num, 0U); ASSERT_GT(int_num, 0U);
TablePropertiesCollection ptc; TablePropertiesCollection ptc;
ASSERT_OK( ASSERT_OK(reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(
reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc)); &ptc));
ASSERT_EQ(1U, ptc.size()); ASSERT_EQ(1U, ptc.size());
auto row = ptc.begin(); auto row = ptc.begin();
auto tp = row->second; auto tp = row->second;
if (full_scan_mode) { if (full_scan_mode) {
// Does not support Get/Seek // Does not support Get/Seek
std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ReadOptions())); std::unique_ptr<Iterator> iter(
iter->SeekToFirst(); dbfull()->NewIterator(ReadOptions()));
ASSERT_TRUE(iter->Valid()); iter->SeekToFirst();
ASSERT_EQ("0000000000000bar", iter->key().ToString()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ("v2", iter->value().ToString()); ASSERT_EQ("0000000000000bar", iter->key().ToString());
iter->Next(); ASSERT_EQ("v2", iter->value().ToString());
ASSERT_TRUE(iter->Valid()); iter->Next();
ASSERT_EQ("1000000000000foo", iter->key().ToString()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ("v3", iter->value().ToString()); ASSERT_EQ("1000000000000foo", iter->key().ToString());
iter->Next(); ASSERT_EQ("v3", iter->value().ToString());
ASSERT_TRUE(!iter->Valid()); iter->Next();
ASSERT_TRUE(iter->status().ok()); ASSERT_TRUE(!iter->Valid());
} else { ASSERT_TRUE(iter->status().ok());
if (!store_index_in_file) {
ASSERT_EQ(total_order ? "4" : "12",
(tp->user_collected_properties)
.at("plain_table_hash_table_size"));
ASSERT_EQ("0", (tp->user_collected_properties)
.at("plain_table_sub_index_size"));
} else { } else {
ASSERT_EQ("0", (tp->user_collected_properties) if (!store_index_in_file) {
.at("plain_table_hash_table_size")); ASSERT_EQ(total_order ? "4" : "12",
ASSERT_EQ("0", (tp->user_collected_properties) (tp->user_collected_properties)
.at("plain_table_sub_index_size")); .at("plain_table_hash_table_size"));
ASSERT_EQ("0", (tp->user_collected_properties)
.at("plain_table_sub_index_size"));
} else {
ASSERT_EQ("0", (tp->user_collected_properties)
.at("plain_table_hash_table_size"));
ASSERT_EQ("0", (tp->user_collected_properties)
.at("plain_table_sub_index_size"));
}
ASSERT_EQ("v3", Get("1000000000000foo"));
ASSERT_EQ("v2", Get("0000000000000bar"));
} }
ASSERT_EQ("v3", Get("1000000000000foo"));
ASSERT_EQ("v2", Get("0000000000000bar"));
} }
} }
}
} }
} }
} }
@ -550,79 +545,79 @@ TEST_P(PlainTableDBTest, Flush2) {
for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024; for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
huge_page_tlb_size += 2 * 1024 * 1024) { huge_page_tlb_size += 2 * 1024 * 1024) {
for (EncodingType encoding_type : {kPlain, kPrefix}) { for (EncodingType encoding_type : {kPlain, kPrefix}) {
for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) { for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
for (int total_order = 0; total_order <= 1; total_order++) { for (int total_order = 0; total_order <= 1; total_order++) {
for (int store_index_in_file = 0; store_index_in_file <= 1; for (int store_index_in_file = 0; store_index_in_file <= 1;
++store_index_in_file) { ++store_index_in_file) {
if (encoding_type == kPrefix && total_order) { if (encoding_type == kPrefix && total_order) {
continue; continue;
} }
if (!bloom_bits && store_index_in_file) { if (!bloom_bits && store_index_in_file) {
continue; continue;
} }
if (total_order && store_index_in_file) { if (total_order && store_index_in_file) {
continue; continue;
} }
bool expect_bloom_not_match = false; bool expect_bloom_not_match = false;
Options options = CurrentOptions(); Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
// Set only one bucket to force bucket conflict. // Set only one bucket to force bucket conflict.
// Test index interval for the same prefix to be 1, 2 and 4 // Test index interval for the same prefix to be 1, 2 and 4
PlainTableOptions plain_table_options; PlainTableOptions plain_table_options;
if (total_order) { if (total_order) {
options.prefix_extractor = nullptr; options.prefix_extractor = nullptr;
plain_table_options.hash_table_ratio = 0; plain_table_options.hash_table_ratio = 0;
plain_table_options.index_sparseness = 2; plain_table_options.index_sparseness = 2;
} else { } else {
plain_table_options.hash_table_ratio = 0.75; plain_table_options.hash_table_ratio = 0.75;
plain_table_options.index_sparseness = 16; plain_table_options.index_sparseness = 16;
} }
plain_table_options.user_key_len = kPlainTableVariableLength; plain_table_options.user_key_len = kPlainTableVariableLength;
plain_table_options.bloom_bits_per_key = bloom_bits; plain_table_options.bloom_bits_per_key = bloom_bits;
plain_table_options.huge_page_tlb_size = huge_page_tlb_size; plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
plain_table_options.encoding_type = encoding_type; plain_table_options.encoding_type = encoding_type;
plain_table_options.store_index_in_file = store_index_in_file; plain_table_options.store_index_in_file = store_index_in_file;
options.table_factory.reset(new TestPlainTableFactory( options.table_factory.reset(new TestPlainTableFactory(
&expect_bloom_not_match, plain_table_options, &expect_bloom_not_match, plain_table_options,
0 /* column_family_id */, kDefaultColumnFamilyName)); 0 /* column_family_id */, kDefaultColumnFamilyName));
DestroyAndReopen(&options); DestroyAndReopen(&options);
ASSERT_OK(Put("0000000000000bar", "b")); ASSERT_OK(Put("0000000000000bar", "b"));
ASSERT_OK(Put("1000000000000foo", "v1")); ASSERT_OK(Put("1000000000000foo", "v1"));
ASSERT_OK(dbfull()->TEST_FlushMemTable()); ASSERT_OK(dbfull()->TEST_FlushMemTable());
ASSERT_OK(Put("1000000000000foo", "v2")); ASSERT_OK(Put("1000000000000foo", "v2"));
ASSERT_OK(dbfull()->TEST_FlushMemTable()); ASSERT_OK(dbfull()->TEST_FlushMemTable());
ASSERT_EQ("v2", Get("1000000000000foo")); ASSERT_EQ("v2", Get("1000000000000foo"));
ASSERT_OK(Put("0000000000000eee", "v3")); ASSERT_OK(Put("0000000000000eee", "v3"));
ASSERT_OK(dbfull()->TEST_FlushMemTable()); ASSERT_OK(dbfull()->TEST_FlushMemTable());
ASSERT_EQ("v3", Get("0000000000000eee")); ASSERT_EQ("v3", Get("0000000000000eee"));
ASSERT_OK(Delete("0000000000000bar")); ASSERT_OK(Delete("0000000000000bar"));
ASSERT_OK(dbfull()->TEST_FlushMemTable()); ASSERT_OK(dbfull()->TEST_FlushMemTable());
ASSERT_EQ("NOT_FOUND", Get("0000000000000bar")); ASSERT_EQ("NOT_FOUND", Get("0000000000000bar"));
ASSERT_OK(Put("0000000000000eee", "v5")); ASSERT_OK(Put("0000000000000eee", "v5"));
ASSERT_OK(Put("9000000000000eee", "v5")); ASSERT_OK(Put("9000000000000eee", "v5"));
ASSERT_OK(dbfull()->TEST_FlushMemTable()); ASSERT_OK(dbfull()->TEST_FlushMemTable());
ASSERT_EQ("v5", Get("0000000000000eee")); ASSERT_EQ("v5", Get("0000000000000eee"));
// Test Bloom Filter // Test Bloom Filter
if (bloom_bits > 0) { if (bloom_bits > 0) {
// Neither key nor value should exist. // Neither key nor value should exist.
expect_bloom_not_match = true; expect_bloom_not_match = true;
ASSERT_EQ("NOT_FOUND", Get("5_not00000000bar")); ASSERT_EQ("NOT_FOUND", Get("5_not00000000bar"));
// Key doesn't exist any more but prefix exists. // Key doesn't exist any more but prefix exists.
if (total_order) { if (total_order) {
ASSERT_EQ("NOT_FOUND", Get("1000000000000not")); ASSERT_EQ("NOT_FOUND", Get("1000000000000not"));
ASSERT_EQ("NOT_FOUND", Get("0000000000000not")); ASSERT_EQ("NOT_FOUND", Get("0000000000000not"));
}
expect_bloom_not_match = false;
}
} }
expect_bloom_not_match = false;
} }
} }
}
}
} }
} }
} }
@ -675,129 +670,129 @@ TEST_P(PlainTableDBTest, Iterator) {
for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024; for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
huge_page_tlb_size += 2 * 1024 * 1024) { huge_page_tlb_size += 2 * 1024 * 1024) {
for (EncodingType encoding_type : {kPlain, kPrefix}) { for (EncodingType encoding_type : {kPlain, kPrefix}) {
for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) { for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
for (int total_order = 0; total_order <= 1; total_order++) { for (int total_order = 0; total_order <= 1; total_order++) {
if (encoding_type == kPrefix && total_order == 1) { if (encoding_type == kPrefix && total_order == 1) {
continue; continue;
} }
bool expect_bloom_not_match = false; bool expect_bloom_not_match = false;
Options options = CurrentOptions(); Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
// Set only one bucket to force bucket conflict. // Set only one bucket to force bucket conflict.
// Test index interval for the same prefix to be 1, 2 and 4 // Test index interval for the same prefix to be 1, 2 and 4
if (total_order) { if (total_order) {
options.prefix_extractor = nullptr; options.prefix_extractor = nullptr;
PlainTableOptions plain_table_options; PlainTableOptions plain_table_options;
plain_table_options.user_key_len = 16; plain_table_options.user_key_len = 16;
plain_table_options.bloom_bits_per_key = bloom_bits; plain_table_options.bloom_bits_per_key = bloom_bits;
plain_table_options.hash_table_ratio = 0; plain_table_options.hash_table_ratio = 0;
plain_table_options.index_sparseness = 2; plain_table_options.index_sparseness = 2;
plain_table_options.huge_page_tlb_size = huge_page_tlb_size; plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
plain_table_options.encoding_type = encoding_type; plain_table_options.encoding_type = encoding_type;
options.table_factory.reset(new TestPlainTableFactory( options.table_factory.reset(new TestPlainTableFactory(
&expect_bloom_not_match, plain_table_options, &expect_bloom_not_match, plain_table_options,
0 /* column_family_id */, kDefaultColumnFamilyName)); 0 /* column_family_id */, kDefaultColumnFamilyName));
} else { } else {
PlainTableOptions plain_table_options; PlainTableOptions plain_table_options;
plain_table_options.user_key_len = 16; plain_table_options.user_key_len = 16;
plain_table_options.bloom_bits_per_key = bloom_bits; plain_table_options.bloom_bits_per_key = bloom_bits;
plain_table_options.hash_table_ratio = 0.75; plain_table_options.hash_table_ratio = 0.75;
plain_table_options.index_sparseness = 16; plain_table_options.index_sparseness = 16;
plain_table_options.huge_page_tlb_size = huge_page_tlb_size; plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
plain_table_options.encoding_type = encoding_type; plain_table_options.encoding_type = encoding_type;
options.table_factory.reset(new TestPlainTableFactory( options.table_factory.reset(new TestPlainTableFactory(
&expect_bloom_not_match, plain_table_options, &expect_bloom_not_match, plain_table_options,
0 /* column_family_id */, kDefaultColumnFamilyName)); 0 /* column_family_id */, kDefaultColumnFamilyName));
} }
DestroyAndReopen(&options); DestroyAndReopen(&options);
ASSERT_OK(Put("1000000000foo002", "v_2")); ASSERT_OK(Put("1000000000foo002", "v_2"));
ASSERT_OK(Put("0000000000000bar", "random")); ASSERT_OK(Put("0000000000000bar", "random"));
ASSERT_OK(Put("1000000000foo001", "v1")); ASSERT_OK(Put("1000000000foo001", "v1"));
ASSERT_OK(Put("3000000000000bar", "bar_v")); ASSERT_OK(Put("3000000000000bar", "bar_v"));
ASSERT_OK(Put("1000000000foo003", "v__3")); ASSERT_OK(Put("1000000000foo003", "v__3"));
ASSERT_OK(Put("1000000000foo004", "v__4")); ASSERT_OK(Put("1000000000foo004", "v__4"));
ASSERT_OK(Put("1000000000foo005", "v__5")); ASSERT_OK(Put("1000000000foo005", "v__5"));
ASSERT_OK(Put("1000000000foo007", "v__7")); ASSERT_OK(Put("1000000000foo007", "v__7"));
ASSERT_OK(Put("1000000000foo008", "v__8")); ASSERT_OK(Put("1000000000foo008", "v__8"));
ASSERT_OK(dbfull()->TEST_FlushMemTable()); ASSERT_OK(dbfull()->TEST_FlushMemTable());
ASSERT_EQ("v1", Get("1000000000foo001")); ASSERT_EQ("v1", Get("1000000000foo001"));
ASSERT_EQ("v__3", Get("1000000000foo003")); ASSERT_EQ("v__3", Get("1000000000foo003"));
Iterator* iter = dbfull()->NewIterator(ReadOptions()); Iterator* iter = dbfull()->NewIterator(ReadOptions());
iter->Seek("1000000000foo000"); iter->Seek("1000000000foo000");
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ("1000000000foo001", iter->key().ToString()); ASSERT_EQ("1000000000foo001", iter->key().ToString());
ASSERT_EQ("v1", iter->value().ToString()); ASSERT_EQ("v1", iter->value().ToString());
iter->Next(); iter->Next();
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ("1000000000foo002", iter->key().ToString()); ASSERT_EQ("1000000000foo002", iter->key().ToString());
ASSERT_EQ("v_2", iter->value().ToString()); ASSERT_EQ("v_2", iter->value().ToString());
iter->Next(); iter->Next();
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ("1000000000foo003", iter->key().ToString()); ASSERT_EQ("1000000000foo003", iter->key().ToString());
ASSERT_EQ("v__3", iter->value().ToString()); ASSERT_EQ("v__3", iter->value().ToString());
iter->Next(); iter->Next();
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ("1000000000foo004", iter->key().ToString()); ASSERT_EQ("1000000000foo004", iter->key().ToString());
ASSERT_EQ("v__4", iter->value().ToString()); ASSERT_EQ("v__4", iter->value().ToString());
iter->Seek("3000000000000bar"); iter->Seek("3000000000000bar");
ASSERT_TRUE(iter->Valid());
ASSERT_EQ("3000000000000bar", iter->key().ToString());
ASSERT_EQ("bar_v", iter->value().ToString());
iter->Seek("1000000000foo000");
ASSERT_TRUE(iter->Valid());
ASSERT_EQ("1000000000foo001", iter->key().ToString());
ASSERT_EQ("v1", iter->value().ToString());
iter->Seek("1000000000foo005");
ASSERT_TRUE(iter->Valid());
ASSERT_EQ("1000000000foo005", iter->key().ToString());
ASSERT_EQ("v__5", iter->value().ToString());
iter->Seek("1000000000foo006");
ASSERT_TRUE(iter->Valid());
ASSERT_EQ("1000000000foo007", iter->key().ToString());
ASSERT_EQ("v__7", iter->value().ToString());
iter->Seek("1000000000foo008");
ASSERT_TRUE(iter->Valid());
ASSERT_EQ("1000000000foo008", iter->key().ToString());
ASSERT_EQ("v__8", iter->value().ToString());
if (total_order == 0) {
iter->Seek("1000000000foo009");
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ("3000000000000bar", iter->key().ToString()); ASSERT_EQ("3000000000000bar", iter->key().ToString());
} ASSERT_EQ("bar_v", iter->value().ToString());
// Test Bloom Filter iter->Seek("1000000000foo000");
if (bloom_bits > 0) { ASSERT_TRUE(iter->Valid());
if (!total_order) { ASSERT_EQ("1000000000foo001", iter->key().ToString());
// Neither key nor value should exist. ASSERT_EQ("v1", iter->value().ToString());
expect_bloom_not_match = true;
iter->Seek("2not000000000bar"); iter->Seek("1000000000foo005");
ASSERT_TRUE(!iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ("NOT_FOUND", Get("2not000000000bar")); ASSERT_EQ("1000000000foo005", iter->key().ToString());
expect_bloom_not_match = false; ASSERT_EQ("v__5", iter->value().ToString());
} else {
expect_bloom_not_match = true; iter->Seek("1000000000foo006");
ASSERT_EQ("NOT_FOUND", Get("2not000000000bar")); ASSERT_TRUE(iter->Valid());
expect_bloom_not_match = false; ASSERT_EQ("1000000000foo007", iter->key().ToString());
ASSERT_EQ("v__7", iter->value().ToString());
iter->Seek("1000000000foo008");
ASSERT_TRUE(iter->Valid());
ASSERT_EQ("1000000000foo008", iter->key().ToString());
ASSERT_EQ("v__8", iter->value().ToString());
if (total_order == 0) {
iter->Seek("1000000000foo009");
ASSERT_TRUE(iter->Valid());
ASSERT_EQ("3000000000000bar", iter->key().ToString());
}
// Test Bloom Filter
if (bloom_bits > 0) {
if (!total_order) {
// Neither key nor value should exist.
expect_bloom_not_match = true;
iter->Seek("2not000000000bar");
ASSERT_TRUE(!iter->Valid());
ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
expect_bloom_not_match = false;
} else {
expect_bloom_not_match = true;
ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
expect_bloom_not_match = false;
}
} }
ASSERT_OK(iter->status());
delete iter;
} }
ASSERT_OK(iter->status());
delete iter;
} }
} }
}
} }
} }
@ -863,7 +858,7 @@ namespace {
std::string MakeLongKey(size_t length, char c) { std::string MakeLongKey(size_t length, char c) {
return std::string(length, c); return std::string(length, c);
} }
} // namespace } // anonymous namespace
TEST_P(PlainTableDBTest, IteratorLargeKeys) { TEST_P(PlainTableDBTest, IteratorLargeKeys) {
Options options = CurrentOptions(); Options options = CurrentOptions();
@ -878,15 +873,10 @@ TEST_P(PlainTableDBTest, IteratorLargeKeys) {
options.prefix_extractor.reset(); options.prefix_extractor.reset();
DestroyAndReopen(&options); DestroyAndReopen(&options);
std::string key_list[] = { std::string key_list[] = {MakeLongKey(30, '0'), MakeLongKey(16, '1'),
MakeLongKey(30, '0'), MakeLongKey(32, '2'), MakeLongKey(60, '3'),
MakeLongKey(16, '1'), MakeLongKey(90, '4'), MakeLongKey(50, '5'),
MakeLongKey(32, '2'), MakeLongKey(26, '6')};
MakeLongKey(60, '3'),
MakeLongKey(90, '4'),
MakeLongKey(50, '5'),
MakeLongKey(26, '6')
};
for (size_t i = 0; i < 7; i++) { for (size_t i = 0; i < 7; i++) {
ASSERT_OK(Put(key_list[i], std::to_string(i))); ASSERT_OK(Put(key_list[i], std::to_string(i)));
@ -913,7 +903,7 @@ namespace {
std::string MakeLongKeyWithPrefix(size_t length, char c) { std::string MakeLongKeyWithPrefix(size_t length, char c) {
return "00000000" + std::string(length - 8, c); return "00000000" + std::string(length - 8, c);
} }
} // namespace } // anonymous namespace
TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) { TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
Options options = CurrentOptions(); Options options = CurrentOptions();
@ -1275,7 +1265,7 @@ TEST_P(PlainTableDBTest, CompactionTrigger) {
Random rnd(301); Random rnd(301);
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1; for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
num++) { num++) {
std::vector<std::string> values; std::vector<std::string> values;
// Write 120KB (10 values, each 12K) // Write 120KB (10 values, each 12K)
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
@ -1287,7 +1277,7 @@ TEST_P(PlainTableDBTest, CompactionTrigger) {
ASSERT_EQ(NumTableFilesAtLevel(0), num + 1); ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
} }
//generate one more file in level-0, and should trigger level-0 compaction // generate one more file in level-0, and should trigger level-0 compaction
std::vector<std::string> values; std::vector<std::string> values;
for (int i = 0; i < 12; i++) { for (int i = 0; i < 12; i++) {
values.push_back(rnd.RandomString(10000)); values.push_back(rnd.RandomString(10000));
@ -1315,8 +1305,7 @@ TEST_P(PlainTableDBTest, AdaptiveTable) {
options.create_if_missing = false; options.create_if_missing = false;
std::shared_ptr<TableFactory> block_based_factory( std::shared_ptr<TableFactory> block_based_factory(
NewBlockBasedTableFactory()); NewBlockBasedTableFactory());
std::shared_ptr<TableFactory> plain_table_factory( std::shared_ptr<TableFactory> plain_table_factory(NewPlainTableFactory());
NewPlainTableFactory());
std::shared_ptr<TableFactory> dummy_factory; std::shared_ptr<TableFactory> dummy_factory;
options.table_factory.reset(NewAdaptiveTableFactory( options.table_factory.reset(NewAdaptiveTableFactory(
block_based_factory, block_based_factory, plain_table_factory)); block_based_factory, block_based_factory, plain_table_factory));

@ -69,7 +69,7 @@ struct TestKey {
}; };
// return a slice backed by test_key // return a slice backed by test_key
inline Slice TestKeyToSlice(std::string &s, const TestKey& test_key) { inline Slice TestKeyToSlice(std::string& s, const TestKey& test_key) {
s.clear(); s.clear();
PutFixed64(&s, test_key.prefix); PutFixed64(&s, test_key.prefix);
PutFixed64(&s, test_key.sorted); PutFixed64(&s, test_key.sorted);
@ -77,20 +77,18 @@ inline Slice TestKeyToSlice(std::string &s, const TestKey& test_key) {
} }
inline const TestKey SliceToTestKey(const Slice& slice) { inline const TestKey SliceToTestKey(const Slice& slice) {
return TestKey(DecodeFixed64(slice.data()), return TestKey(DecodeFixed64(slice.data()), DecodeFixed64(slice.data() + 8));
DecodeFixed64(slice.data() + 8));
} }
class TestKeyComparator : public Comparator { class TestKeyComparator : public Comparator {
public: public:
// Compare needs to be aware of the possibility of a and/or b is // Compare needs to be aware of the possibility of a and/or b is
// prefix only // prefix only
int Compare(const Slice& a, const Slice& b) const override { int Compare(const Slice& a, const Slice& b) const override {
const TestKey kkey_a = SliceToTestKey(a); const TestKey kkey_a = SliceToTestKey(a);
const TestKey kkey_b = SliceToTestKey(b); const TestKey kkey_b = SliceToTestKey(b);
const TestKey *key_a = &kkey_a; const TestKey* key_a = &kkey_a;
const TestKey *key_b = &kkey_b; const TestKey* key_b = &kkey_b;
if (key_a->prefix != key_b->prefix) { if (key_a->prefix != key_b->prefix) {
if (key_a->prefix < key_b->prefix) return -1; if (key_a->prefix < key_b->prefix) return -1;
if (key_a->prefix > key_b->prefix) return 1; if (key_a->prefix > key_b->prefix) return 1;
@ -215,7 +213,7 @@ class SamePrefixTransform : public SliceTransform {
bool FullLengthEnabled(size_t* /*len*/) const override { return false; } bool FullLengthEnabled(size_t* /*len*/) const override { return false; }
}; };
} // namespace } // anonymous namespace
class PrefixTest : public testing::Test { class PrefixTest : public testing::Test {
public: public:
@ -226,7 +224,7 @@ class PrefixTest : public testing::Test {
options.write_buffer_size = FLAGS_write_buffer_size; options.write_buffer_size = FLAGS_write_buffer_size;
options.max_write_buffer_number = FLAGS_max_write_buffer_number; options.max_write_buffer_number = FLAGS_max_write_buffer_number;
options.min_write_buffer_number_to_merge = options.min_write_buffer_number_to_merge =
FLAGS_min_write_buffer_number_to_merge; FLAGS_min_write_buffer_number_to_merge;
options.memtable_prefix_bloom_size_ratio = options.memtable_prefix_bloom_size_ratio =
FLAGS_memtable_prefix_bloom_size_ratio; FLAGS_memtable_prefix_bloom_size_ratio;
@ -239,21 +237,19 @@ class PrefixTest : public testing::Test {
options.table_factory.reset(NewBlockBasedTableFactory(bbto)); options.table_factory.reset(NewBlockBasedTableFactory(bbto));
options.allow_concurrent_memtable_write = false; options.allow_concurrent_memtable_write = false;
Status s = DB::Open(options, kDbName, &db); Status s = DB::Open(options, kDbName, &db);
EXPECT_OK(s); EXPECT_OK(s);
return std::shared_ptr<DB>(db); return std::shared_ptr<DB>(db);
} }
void FirstOption() { void FirstOption() { option_config_ = kBegin; }
option_config_ = kBegin;
}
bool NextOptions(int bucket_count) { bool NextOptions(int bucket_count) {
// skip some options // skip some options
option_config_++; option_config_++;
if (option_config_ < kEnd) { if (option_config_ < kEnd) {
options.prefix_extractor.reset(NewFixedPrefixTransform(8)); options.prefix_extractor.reset(NewFixedPrefixTransform(8));
switch(option_config_) { switch (option_config_) {
case kHashSkipList: case kHashSkipList:
options.memtable_factory.reset( options.memtable_factory.reset(
NewHashSkipListRepFactory(bucket_count, FLAGS_skiplist_height)); NewHashSkipListRepFactory(bucket_count, FLAGS_skiplist_height));
@ -350,8 +346,7 @@ TEST_F(PrefixTest, TestResult) {
FirstOption(); FirstOption();
while (NextOptions(num_buckets)) { while (NextOptions(num_buckets)) {
std::cout << "*** Mem table: " << options.memtable_factory->Name() std::cout << "*** Mem table: " << options.memtable_factory->Name()
<< " number of buckets: " << num_buckets << " number of buckets: " << num_buckets << std::endl;
<< std::endl;
ASSERT_OK(DestroyDB(kDbName, Options())); ASSERT_OK(DestroyDB(kDbName, Options()));
auto db = OpenDb(); auto db = OpenDb();
WriteOptions write_options; WriteOptions write_options;
@ -581,7 +576,7 @@ TEST_F(PrefixTest, PrefixValid) {
TEST_F(PrefixTest, DynamicPrefixIterator) { TEST_F(PrefixTest, DynamicPrefixIterator) {
while (NextOptions(FLAGS_bucket_count)) { while (NextOptions(FLAGS_bucket_count)) {
std::cout << "*** Mem table: " << options.memtable_factory->Name() std::cout << "*** Mem table: " << options.memtable_factory->Name()
<< std::endl; << std::endl;
ASSERT_OK(DestroyDB(kDbName, Options())); ASSERT_OK(DestroyDB(kDbName, Options()));
auto db = OpenDb(); auto db = OpenDb();
WriteOptions write_options; WriteOptions write_options;
@ -600,7 +595,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
HistogramImpl hist_put_comparison; HistogramImpl hist_put_comparison;
// insert x random prefix, each with y continuous element. // insert x random prefix, each with y continuous element.
for (auto prefix : prefixes) { for (auto prefix : prefixes) {
for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) { for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) {
TestKey test_key(prefix, sorted); TestKey test_key(prefix, sorted);
std::string s; std::string s;
@ -615,8 +610,9 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
} }
} }
std::cout << "Put key comparison: \n" << hist_put_comparison.ToString() std::cout << "Put key comparison: \n"
<< "Put time: \n" << hist_put_time.ToString(); << hist_put_comparison.ToString() << "Put time: \n"
<< hist_put_time.ToString();
// test seek existing keys // test seek existing keys
HistogramImpl hist_seek_time; HistogramImpl hist_seek_time;
@ -635,8 +631,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
auto key_prefix = options.prefix_extractor->Transform(key); auto key_prefix = options.prefix_extractor->Transform(key);
uint64_t total_keys = 0; uint64_t total_keys = 0;
for (iter->Seek(key); for (iter->Seek(key);
iter->Valid() && iter->key().starts_with(key_prefix); iter->Valid() && iter->key().starts_with(key_prefix); iter->Next()) {
iter->Next()) {
if (FLAGS_trigger_deadlock) { if (FLAGS_trigger_deadlock) {
std::cout << "Behold the deadlock!\n"; std::cout << "Behold the deadlock!\n";
db->Delete(write_options, iter->key()); db->Delete(write_options, iter->key());
@ -645,12 +640,12 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
} }
hist_seek_time.Add(timer.ElapsedNanos()); hist_seek_time.Add(timer.ElapsedNanos());
hist_seek_comparison.Add(get_perf_context()->user_key_comparison_count); hist_seek_comparison.Add(get_perf_context()->user_key_comparison_count);
ASSERT_EQ(total_keys, FLAGS_items_per_prefix - FLAGS_items_per_prefix/2); ASSERT_EQ(total_keys,
FLAGS_items_per_prefix - FLAGS_items_per_prefix / 2);
} }
std::cout << "Seek key comparison: \n" std::cout << "Seek key comparison: \n"
<< hist_seek_comparison.ToString() << hist_seek_comparison.ToString() << "Seek time: \n"
<< "Seek time: \n"
<< hist_seek_time.ToString(); << hist_seek_time.ToString();
// test non-existing keys // test non-existing keys
@ -658,8 +653,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
HistogramImpl hist_no_seek_comparison; HistogramImpl hist_no_seek_comparison;
for (auto prefix = FLAGS_total_prefixes; for (auto prefix = FLAGS_total_prefixes;
prefix < FLAGS_total_prefixes + 10000; prefix < FLAGS_total_prefixes + 10000; prefix++) {
prefix++) {
TestKey test_key(prefix, 0); TestKey test_key(prefix, 0);
std::string s; std::string s;
Slice key = TestKeyToSlice(s, test_key); Slice key = TestKeyToSlice(s, test_key);
@ -668,7 +662,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
StopWatchNano timer(SystemClock::Default().get(), true); StopWatchNano timer(SystemClock::Default().get(), true);
iter->Seek(key); iter->Seek(key);
hist_no_seek_time.Add(timer.ElapsedNanos()); hist_no_seek_time.Add(timer.ElapsedNanos());
hist_no_seek_comparison.Add(get_perf_context()->user_key_comparison_count); hist_no_seek_comparison.Add(
get_perf_context()->user_key_comparison_count);
ASSERT_TRUE(!iter->Valid()); ASSERT_TRUE(!iter->Valid());
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
} }

@ -502,7 +502,7 @@ class TruncatedRangeDelMergingIter : public InternalIterator {
size_t ts_sz_; size_t ts_sz_;
}; };
} // namespace } // anonymous namespace
std::unique_ptr<FragmentedRangeTombstoneIterator> std::unique_ptr<FragmentedRangeTombstoneIterator>
CompactionRangeDelAggregator::NewIterator(const Slice* lower_bound, CompactionRangeDelAggregator::NewIterator(const Slice* lower_bound,

@ -192,7 +192,7 @@ void VerifyFragmentedRangeDels(
EXPECT_FALSE(iter->Valid()); EXPECT_FALSE(iter->Valid());
} }
} // namespace } // anonymous namespace
TEST_F(RangeDelAggregatorTest, EmptyTruncatedIter) { TEST_F(RangeDelAggregatorTest, EmptyTruncatedIter) {
auto range_del_iter = MakeRangeDelIter({}); auto range_del_iter = MakeRangeDelIter({});

@ -281,7 +281,7 @@ class Repairer {
std::vector<std::string> to_search_paths; std::vector<std::string> to_search_paths;
for (size_t path_id = 0; path_id < db_options_.db_paths.size(); path_id++) { for (size_t path_id = 0; path_id < db_options_.db_paths.size(); path_id++) {
to_search_paths.push_back(db_options_.db_paths[path_id].path); to_search_paths.push_back(db_options_.db_paths[path_id].path);
} }
// search wal_dir if user uses a customize wal_dir // search wal_dir if user uses a customize wal_dir
@ -332,7 +332,8 @@ class Repairer {
void ConvertLogFilesToTables() { void ConvertLogFilesToTables() {
const auto& wal_dir = immutable_db_options_.GetWalDir(); const auto& wal_dir = immutable_db_options_.GetWalDir();
for (size_t i = 0; i < logs_.size(); i++) { for (size_t i = 0; i < logs_.size(); i++) {
// we should use LogFileName(wal_dir, logs_[i]) here. user might uses wal_dir option. // we should use LogFileName(wal_dir, logs_[i]) here. user might uses
// wal_dir option.
std::string logname = LogFileName(wal_dir, logs_[i]); std::string logname = LogFileName(wal_dir, logs_[i]);
Status status = ConvertLogToTable(wal_dir, logs_[i]); Status status = ConvertLogToTable(wal_dir, logs_[i]);
if (!status.ok()) { if (!status.ok()) {
@ -393,8 +394,8 @@ class Repairer {
int counter = 0; int counter = 0;
while (reader.ReadRecord(&record, &scratch)) { while (reader.ReadRecord(&record, &scratch)) {
if (record.size() < WriteBatchInternal::kHeader) { if (record.size() < WriteBatchInternal::kHeader) {
reporter.Corruption( reporter.Corruption(record.size(),
record.size(), Status::Corruption("log record too small")); Status::Corruption("log record too small"));
continue; continue;
} }
Status record_status = WriteBatchInternal::SetContents(&batch, record); Status record_status = WriteBatchInternal::SetContents(&batch, record);
@ -715,8 +716,7 @@ Status GetDefaultCFOptions(
} // anonymous namespace } // anonymous namespace
Status RepairDB(const std::string& dbname, const DBOptions& db_options, Status RepairDB(const std::string& dbname, const DBOptions& db_options,
const std::vector<ColumnFamilyDescriptor>& column_families const std::vector<ColumnFamilyDescriptor>& column_families) {
) {
ColumnFamilyOptions default_cf_opts; ColumnFamilyOptions default_cf_opts;
Status status = GetDefaultCFOptions(column_families, &default_cf_opts); Status status = GetDefaultCFOptions(column_families, &default_cf_opts);
if (!status.ok()) { if (!status.ok()) {
@ -756,8 +756,7 @@ Status RepairDB(const std::string& dbname, const Options& options) {
DBOptions db_options(opts); DBOptions db_options(opts);
ColumnFamilyOptions cf_options(opts); ColumnFamilyOptions cf_options(opts);
Repairer repairer(dbname, db_options, Repairer repairer(dbname, db_options, {}, cf_options /* default_cf_opts */,
{}, cf_options /* default_cf_opts */,
cf_options /* unknown_cf_opts */, cf_options /* unknown_cf_opts */,
true /* create_unknown_cfs */); true /* create_unknown_cfs */);
Status status = repairer.Run(); Status status = repairer.Run();

@ -279,7 +279,7 @@ TEST_F(RepairTest, SeparateWalDir) {
ASSERT_EQ(total_ssts_size, 0); ASSERT_EQ(total_ssts_size, 0);
} }
std::string manifest_path = std::string manifest_path =
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
Close(); Close();
ASSERT_OK(env_->FileExists(manifest_path)); ASSERT_OK(env_->FileExists(manifest_path));
@ -301,7 +301,7 @@ TEST_F(RepairTest, SeparateWalDir) {
ASSERT_EQ(Get("key"), "val"); ASSERT_EQ(Get("key"), "val");
ASSERT_EQ(Get("foo"), "bar"); ASSERT_EQ(Get("foo"), "bar");
} while(ChangeWalOptions()); } while (ChangeWalOptions());
} }
TEST_F(RepairTest, RepairMultipleColumnFamilies) { TEST_F(RepairTest, RepairMultipleColumnFamilies) {
@ -387,8 +387,7 @@ TEST_F(RepairTest, RepairColumnFamilyOptions) {
ASSERT_EQ(fname_to_props.size(), 2U); ASSERT_EQ(fname_to_props.size(), 2U);
for (const auto& fname_and_props : fname_to_props) { for (const auto& fname_and_props : fname_to_props) {
std::string comparator_name(rev_opts.comparator->Name()); std::string comparator_name(rev_opts.comparator->Name());
ASSERT_EQ(comparator_name, ASSERT_EQ(comparator_name, fname_and_props.second->comparator_name);
fname_and_props.second->comparator_name);
} }
Close(); Close();

@ -3,14 +3,13 @@
// COPYING file in the root directory) and Apache 2.0 License // COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#include "rocksdb/snapshot.h"
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "rocksdb/snapshot.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
ManagedSnapshot::ManagedSnapshot(DB* db) : db_(db), ManagedSnapshot::ManagedSnapshot(DB* db)
snapshot_(db->GetSnapshot()) {} : db_(db), snapshot_(db->GetSnapshot()) {}
ManagedSnapshot::ManagedSnapshot(DB* db, const Snapshot* _snapshot) ManagedSnapshot::ManagedSnapshot(DB* db, const Snapshot* _snapshot)
: db_(db), snapshot_(_snapshot) {} : db_(db), snapshot_(_snapshot) {}
@ -21,6 +20,6 @@ ManagedSnapshot::~ManagedSnapshot() {
} }
} }
const Snapshot* ManagedSnapshot::snapshot() { return snapshot_;} const Snapshot* ManagedSnapshot::snapshot() { return snapshot_; }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -41,7 +41,7 @@ class SnapshotImpl : public Snapshot {
SnapshotImpl* prev_; SnapshotImpl* prev_;
SnapshotImpl* next_; SnapshotImpl* next_;
SnapshotList* list_; // just for sanity checks SnapshotList* list_; // just for sanity checks
int64_t unix_time_; int64_t unix_time_;
@ -56,7 +56,7 @@ class SnapshotList {
SnapshotList() { SnapshotList() {
list_.prev_ = &list_; list_.prev_ = &list_;
list_.next_ = &list_; list_.next_ = &list_;
list_.number_ = 0xFFFFFFFFL; // placeholder marker, for debugging list_.number_ = 0xFFFFFFFFL; // placeholder marker, for debugging
// Set all the variables to make UBSAN happy. // Set all the variables to make UBSAN happy.
list_.list_ = nullptr; list_.list_ = nullptr;
list_.unix_time_ = 0; list_.unix_time_ = 0;
@ -72,8 +72,14 @@ class SnapshotList {
assert(list_.next_ != &list_ || 0 == count_); assert(list_.next_ != &list_ || 0 == count_);
return list_.next_ == &list_; return list_.next_ == &list_;
} }
SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; } SnapshotImpl* oldest() const {
SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; } assert(!empty());
return list_.next_;
}
SnapshotImpl* newest() const {
assert(!empty());
return list_.prev_;
}
SnapshotImpl* New(SnapshotImpl* s, SequenceNumber seq, uint64_t unix_time, SnapshotImpl* New(SnapshotImpl* s, SequenceNumber seq, uint64_t unix_time,
bool is_write_conflict_boundary, bool is_write_conflict_boundary,

@ -38,7 +38,7 @@ static void DeleteEntry(const Slice& /*key*/, void* value) {
T* typed_value = reinterpret_cast<T*>(value); T* typed_value = reinterpret_cast<T*>(value);
delete typed_value; delete typed_value;
} }
} // namespace } // anonymous namespace
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
// Generate the regular and coroutine versions of some methods by // Generate the regular and coroutine versions of some methods by
@ -79,7 +79,7 @@ void AppendVarint64(IterKey* key, uint64_t v) {
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
} // namespace } // anonymous namespace
const int kLoadConcurency = 128; const int kLoadConcurency = 128;
@ -103,8 +103,7 @@ TableCache::TableCache(const ImmutableOptions& ioptions,
} }
} }
TableCache::~TableCache() { TableCache::~TableCache() {}
}
TableReader* TableCache::GetTableReaderFromHandle(Cache::Handle* handle) { TableReader* TableCache::GetTableReaderFromHandle(Cache::Handle* handle) {
return reinterpret_cast<TableReader*>(cache_->Value(handle)); return reinterpret_cast<TableReader*>(cache_->Value(handle));

@ -27,7 +27,7 @@ uint64_t GetUint64Property(const UserCollectedProperties& props,
return GetVarint64(&raw, &val) ? val : 0; return GetVarint64(&raw, &val) ? val : 0;
} }
} // namespace } // anonymous namespace
Status UserKeyTablePropertiesCollector::InternalAdd(const Slice& key, Status UserKeyTablePropertiesCollector::InternalAdd(const Slice& key,
const Slice& value, const Slice& value,
@ -54,13 +54,12 @@ Status UserKeyTablePropertiesCollector::Finish(
return collector_->Finish(properties); return collector_->Finish(properties);
} }
UserCollectedProperties UserCollectedProperties UserKeyTablePropertiesCollector::GetReadableProperties()
UserKeyTablePropertiesCollector::GetReadableProperties() const { const {
return collector_->GetReadableProperties(); return collector_->GetReadableProperties();
} }
uint64_t GetDeletedKeys( uint64_t GetDeletedKeys(const UserCollectedProperties& props) {
const UserCollectedProperties& props) {
bool property_present_ignored; bool property_present_ignored;
return GetUint64Property(props, TablePropertiesNames::kDeletedKeys, return GetUint64Property(props, TablePropertiesNames::kDeletedKeys,
&property_present_ignored); &property_present_ignored);
@ -68,8 +67,8 @@ uint64_t GetDeletedKeys(
uint64_t GetMergeOperands(const UserCollectedProperties& props, uint64_t GetMergeOperands(const UserCollectedProperties& props,
bool* property_present) { bool* property_present) {
return GetUint64Property( return GetUint64Property(props, TablePropertiesNames::kMergeOperands,
props, TablePropertiesNames::kMergeOperands, property_present); property_present);
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -61,30 +61,30 @@ void MakeBuilder(
} // namespace } // namespace
// Collects keys that starts with "A" in a table. // Collects keys that starts with "A" in a table.
class RegularKeysStartWithA: public TablePropertiesCollector { class RegularKeysStartWithA : public TablePropertiesCollector {
public: public:
const char* Name() const override { return "RegularKeysStartWithA"; } const char* Name() const override { return "RegularKeysStartWithA"; }
Status Finish(UserCollectedProperties* properties) override { Status Finish(UserCollectedProperties* properties) override {
std::string encoded; std::string encoded;
std::string encoded_num_puts; std::string encoded_num_puts;
std::string encoded_num_deletes; std::string encoded_num_deletes;
std::string encoded_num_single_deletes; std::string encoded_num_single_deletes;
std::string encoded_num_size_changes; std::string encoded_num_size_changes;
PutVarint32(&encoded, count_); PutVarint32(&encoded, count_);
PutVarint32(&encoded_num_puts, num_puts_); PutVarint32(&encoded_num_puts, num_puts_);
PutVarint32(&encoded_num_deletes, num_deletes_); PutVarint32(&encoded_num_deletes, num_deletes_);
PutVarint32(&encoded_num_single_deletes, num_single_deletes_); PutVarint32(&encoded_num_single_deletes, num_single_deletes_);
PutVarint32(&encoded_num_size_changes, num_size_changes_); PutVarint32(&encoded_num_size_changes, num_size_changes_);
*properties = UserCollectedProperties{ *properties = UserCollectedProperties{
{"TablePropertiesTest", message_}, {"TablePropertiesTest", message_},
{"Count", encoded}, {"Count", encoded},
{"NumPuts", encoded_num_puts}, {"NumPuts", encoded_num_puts},
{"NumDeletes", encoded_num_deletes}, {"NumDeletes", encoded_num_deletes},
{"NumSingleDeletes", encoded_num_single_deletes}, {"NumSingleDeletes", encoded_num_single_deletes},
{"NumSizeChanges", encoded_num_size_changes}, {"NumSizeChanges", encoded_num_size_changes},
}; };
return Status::OK(); return Status::OK();
} }
Status AddUserKey(const Slice& user_key, const Slice& /*value*/, Status AddUserKey(const Slice& user_key, const Slice& /*value*/,
@ -338,7 +338,7 @@ void TestCustomizedTablePropertiesCollector(
TEST_P(TablePropertiesTest, CustomizedTablePropertiesCollector) { TEST_P(TablePropertiesTest, CustomizedTablePropertiesCollector) {
// Test properties collectors with internal keys or regular keys // Test properties collectors with internal keys or regular keys
// for block based table // for block based table
for (bool encode_as_internal : { true, false }) { for (bool encode_as_internal : {true, false}) {
Options options; Options options;
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.flush_block_policy_factory = table_options.flush_block_policy_factory =
@ -404,7 +404,7 @@ void TestInternalKeyPropertiesCollector(
// HACK: Set options.info_log to avoid writing log in // HACK: Set options.info_log to avoid writing log in
// SanitizeOptions(). // SanitizeOptions().
options.info_log = std::make_shared<test::NullLogger>(); options.info_log = std::make_shared<test::NullLogger>();
options = SanitizeOptions("db", // just a place holder options = SanitizeOptions("db", // just a place holder
options); options);
ImmutableOptions ioptions(options); ImmutableOptions ioptions(options);
GetIntTblPropCollectorFactory(ioptions, &int_tbl_prop_collector_factories); GetIntTblPropCollectorFactory(ioptions, &int_tbl_prop_collector_factories);

@ -41,7 +41,7 @@ TransactionLogIteratorImpl::TransactionLogIteratorImpl(
current_status_.PermitUncheckedError(); // Clear on start current_status_.PermitUncheckedError(); // Clear on start
reporter_.env = options_->env; reporter_.env = options_->env;
reporter_.info_log = options_->info_log.get(); reporter_.info_log = options_->info_log.get();
SeekToStartSequence(); // Seek till starting sequence SeekToStartSequence(); // Seek till starting sequence
} }
Status TransactionLogIteratorImpl::OpenLogFile( Status TransactionLogIteratorImpl::OpenLogFile(
@ -62,8 +62,7 @@ Status TransactionLogIteratorImpl::OpenLogFile(
// If cannot open file in DB directory. // If cannot open file in DB directory.
// Try the archive dir, as it could have moved in the meanwhile. // Try the archive dir, as it could have moved in the meanwhile.
fname = ArchivedLogFileName(dir_, log_file->LogNumber()); fname = ArchivedLogFileName(dir_, log_file->LogNumber());
s = fs->NewSequentialFile(fname, optimized_env_options, s = fs->NewSequentialFile(fname, optimized_env_options, &file, nullptr);
&file, nullptr);
} }
} }
if (s.ok()) { if (s.ok()) {
@ -74,7 +73,7 @@ Status TransactionLogIteratorImpl::OpenLogFile(
return s; return s;
} }
BatchResult TransactionLogIteratorImpl::GetBatch() { BatchResult TransactionLogIteratorImpl::GetBatch() {
assert(is_valid_); // cannot call in a non valid state. assert(is_valid_); // cannot call in a non valid state.
BatchResult result; BatchResult result;
result.sequence = current_batch_seq_; result.sequence = current_batch_seq_;
@ -124,8 +123,8 @@ void TransactionLogIteratorImpl::SeekToStartSequence(uint64_t start_file_index,
} }
while (RestrictedRead(&record)) { while (RestrictedRead(&record)) {
if (record.size() < WriteBatchInternal::kHeader) { if (record.size() < WriteBatchInternal::kHeader) {
reporter_.Corruption( reporter_.Corruption(record.size(),
record.size(), Status::Corruption("very small log record")); Status::Corruption("very small log record"));
continue; continue;
} }
UpdateCurrentWriteBatch(record); UpdateCurrentWriteBatch(record);
@ -137,11 +136,12 @@ void TransactionLogIteratorImpl::SeekToStartSequence(uint64_t start_file_index,
reporter_.Info(current_status_.ToString().c_str()); reporter_.Info(current_status_.ToString().c_str());
return; return;
} else if (strict) { } else if (strict) {
reporter_.Info("Could seek required sequence number. Iterator will " reporter_.Info(
"continue."); "Could seek required sequence number. Iterator will "
"continue.");
} }
is_valid_ = true; is_valid_ = true;
started_ = true; // set started_ as we could seek till starting sequence started_ = true; // set started_ as we could seek till starting sequence
return; return;
} else { } else {
is_valid_ = false; is_valid_ = false;
@ -182,15 +182,15 @@ void TransactionLogIteratorImpl::NextImpl(bool internal) {
// Runs every time until we can seek to the start sequence // Runs every time until we can seek to the start sequence
SeekToStartSequence(); SeekToStartSequence();
} }
while(true) { while (true) {
assert(current_log_reader_); assert(current_log_reader_);
if (current_log_reader_->IsEOF()) { if (current_log_reader_->IsEOF()) {
current_log_reader_->UnmarkEOF(); current_log_reader_->UnmarkEOF();
} }
while (RestrictedRead(&record)) { while (RestrictedRead(&record)) {
if (record.size() < WriteBatchInternal::kHeader) { if (record.size() < WriteBatchInternal::kHeader) {
reporter_.Corruption( reporter_.Corruption(record.size(),
record.size(), Status::Corruption("very small log record")); Status::Corruption("very small log record"));
continue; continue;
} else { } else {
// started_ should be true if called by application // started_ should be true if called by application

@ -23,12 +23,11 @@ namespace ROCKSDB_NAMESPACE {
class LogFileImpl : public LogFile { class LogFileImpl : public LogFile {
public: public:
LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq, LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
uint64_t sizeBytes) : uint64_t sizeBytes)
logNumber_(logNum), : logNumber_(logNum),
type_(logType), type_(logType),
startSequence_(startSeq), startSequence_(startSeq),
sizeFileBytes_(sizeBytes) { sizeFileBytes_(sizeBytes) {}
}
std::string PathName() const override { std::string PathName() const override {
if (type_ == kArchivedLogFile) { if (type_ == kArchivedLogFile) {
@ -45,7 +44,7 @@ class LogFileImpl : public LogFile {
uint64_t SizeFileBytes() const override { return sizeFileBytes_; } uint64_t SizeFileBytes() const override { return sizeFileBytes_; }
bool operator < (const LogFile& that) const { bool operator<(const LogFile& that) const {
return LogNumber() < that.LogNumber(); return LogNumber() < that.LogNumber();
} }
@ -54,7 +53,6 @@ class LogFileImpl : public LogFile {
WalFileType type_; WalFileType type_;
SequenceNumber startSequence_; SequenceNumber startSequence_;
uint64_t sizeFileBytes_; uint64_t sizeFileBytes_;
}; };
class TransactionLogIteratorImpl : public TransactionLogIterator { class TransactionLogIteratorImpl : public TransactionLogIterator {

@ -6,8 +6,10 @@
#pragma once #pragma once
#include <stdint.h> #include <stdint.h>
#include <atomic> #include <atomic>
#include <mutex> #include <mutex>
#include "util/autovector.h" #include "util/autovector.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -1656,10 +1656,10 @@ TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) {
UpdateVersionStorageInfo(&new_vstorage); UpdateVersionStorageInfo(&new_vstorage);
VersionBuilder version_builder2(env_options, &ioptions_, table_cache, VersionBuilder version_builder2(env_options, &ioptions_, table_cache,
&new_vstorage, version_set); &new_vstorage, version_set);
VersionStorageInfo new_vstorage2(&icmp_, ucmp_, options_.num_levels, VersionStorageInfo new_vstorage2(&icmp_, ucmp_, options_.num_levels,
kCompactionStyleLevel, nullptr, kCompactionStyleLevel, nullptr,
true /* force_consistency_checks */); true /* force_consistency_checks */);
ASSERT_NOK(version_builder2.Apply(&version_edit)); ASSERT_NOK(version_builder2.Apply(&version_edit));
UnrefFilesInVersion(&new_vstorage); UnrefFilesInVersion(&new_vstorage);

@ -20,9 +20,7 @@
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
namespace { namespace {} // anonymous namespace
} // anonymous namespace
uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) { uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) {
assert(number <= kFileNumberMask); assert(number <= kFileNumberMask);
@ -501,8 +499,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
break; break;
case kCompactCursor: case kCompactCursor:
if (GetLevel(&input, &level, &msg) && if (GetLevel(&input, &level, &msg) && GetInternalKey(&input, &key)) {
GetInternalKey(&input, &key)) {
// Here we re-use the output format of compact pointer in LevelDB // Here we re-use the output format of compact pointer in LevelDB
// to persist compact_cursors_ // to persist compact_cursors_
compact_cursors_.push_back(std::make_pair(level, key)); compact_cursors_.push_back(std::make_pair(level, key));

@ -114,7 +114,7 @@ struct FileDescriptor {
// Table reader in table_reader_handle // Table reader in table_reader_handle
TableReader* table_reader; TableReader* table_reader;
uint64_t packed_number_and_path_id; uint64_t packed_number_and_path_id;
uint64_t file_size; // File size in bytes uint64_t file_size; // File size in bytes
SequenceNumber smallest_seqno; // The smallest seqno in this file SequenceNumber smallest_seqno; // The smallest seqno in this file
SequenceNumber largest_seqno; // The largest seqno in this file SequenceNumber largest_seqno; // The largest seqno in this file
@ -146,8 +146,8 @@ struct FileDescriptor {
return packed_number_and_path_id & kFileNumberMask; return packed_number_and_path_id & kFileNumberMask;
} }
uint32_t GetPathId() const { uint32_t GetPathId() const {
return static_cast<uint32_t>( return static_cast<uint32_t>(packed_number_and_path_id /
packed_number_and_path_id / (kFileNumberMask + 1)); (kFileNumberMask + 1));
} }
uint64_t GetFileSize() const { return file_size; } uint64_t GetFileSize() const { return file_size; }
}; };
@ -166,8 +166,8 @@ struct FileSampledStats {
struct FileMetaData { struct FileMetaData {
FileDescriptor fd; FileDescriptor fd;
InternalKey smallest; // Smallest internal key served by table InternalKey smallest; // Smallest internal key served by table
InternalKey largest; // Largest internal key served by table InternalKey largest; // Largest internal key served by table
// Needs to be disposed when refs becomes 0. // Needs to be disposed when refs becomes 0.
Cache::Handle* table_reader_handle = nullptr; Cache::Handle* table_reader_handle = nullptr;
@ -312,15 +312,11 @@ struct FileMetaData {
struct FdWithKeyRange { struct FdWithKeyRange {
FileDescriptor fd; FileDescriptor fd;
FileMetaData* file_metadata; // Point to all metadata FileMetaData* file_metadata; // Point to all metadata
Slice smallest_key; // slice that contain smallest key Slice smallest_key; // slice that contain smallest key
Slice largest_key; // slice that contain largest key Slice largest_key; // slice that contain largest key
FdWithKeyRange() FdWithKeyRange()
: fd(), : fd(), file_metadata(nullptr), smallest_key(), largest_key() {}
file_metadata(nullptr),
smallest_key(),
largest_key() {
}
FdWithKeyRange(FileDescriptor _fd, Slice _smallest_key, Slice _largest_key, FdWithKeyRange(FileDescriptor _fd, Slice _smallest_key, Slice _largest_key,
FileMetaData* _file_metadata) FileMetaData* _file_metadata)

@ -93,23 +93,19 @@ namespace {
// Find File in LevelFilesBrief data structure // Find File in LevelFilesBrief data structure
// Within an index range defined by left and right // Within an index range defined by left and right
int FindFileInRange(const InternalKeyComparator& icmp, int FindFileInRange(const InternalKeyComparator& icmp,
const LevelFilesBrief& file_level, const LevelFilesBrief& file_level, const Slice& key,
const Slice& key, uint32_t left, uint32_t right) {
uint32_t left,
uint32_t right) {
auto cmp = [&](const FdWithKeyRange& f, const Slice& k) -> bool { auto cmp = [&](const FdWithKeyRange& f, const Slice& k) -> bool {
return icmp.InternalKeyComparator::Compare(f.largest_key, k) < 0; return icmp.InternalKeyComparator::Compare(f.largest_key, k) < 0;
}; };
const auto &b = file_level.files; const auto& b = file_level.files;
return static_cast<int>(std::lower_bound(b + left, return static_cast<int>(std::lower_bound(b + left, b + right, key, cmp) - b);
b + right, key, cmp) - b);
} }
Status OverlapWithIterator(const Comparator* ucmp, Status OverlapWithIterator(const Comparator* ucmp,
const Slice& smallest_user_key, const Slice& smallest_user_key,
const Slice& largest_user_key, const Slice& largest_user_key,
InternalIterator* iter, InternalIterator* iter, bool* overlap) {
bool* overlap) {
InternalKey range_start(smallest_user_key, kMaxSequenceNumber, InternalKey range_start(smallest_user_key, kMaxSequenceNumber,
kValueTypeForSeek); kValueTypeForSeek);
iter->Seek(range_start.Encode()); iter->Seek(range_start.Encode());
@ -187,9 +183,9 @@ class FilePicker {
// Do key range filtering of files or/and fractional cascading if: // Do key range filtering of files or/and fractional cascading if:
// (1) not all the files are in level 0, or // (1) not all the files are in level 0, or
// (2) there are more than 3 current level files // (2) there are more than 3 current level files
// If there are only 3 or less current level files in the system, we skip // If there are only 3 or less current level files in the system, we
// the key range filtering. In this case, more likely, the system is // skip the key range filtering. In this case, more likely, the system
// highly tuned to minimize number of tables queried by each query, // is highly tuned to minimize number of tables queried by each query,
// so it is unlikely that key range filtering is more efficient than // so it is unlikely that key range filtering is more efficient than
// querying the files. // querying the files.
if (num_levels_ > 1 || curr_file_level_->num_files > 3) { if (num_levels_ > 1 || curr_file_level_->num_files > 3) {
@ -211,11 +207,9 @@ class FilePicker {
// Setup file search bound for the next level based on the // Setup file search bound for the next level based on the
// comparison results // comparison results
if (curr_level_ > 0) { if (curr_level_ > 0) {
file_indexer_->GetNextLevelIndex(curr_level_, file_indexer_->GetNextLevelIndex(
curr_index_in_curr_level_, curr_level_, curr_index_in_curr_level_, cmp_smallest,
cmp_smallest, cmp_largest, cmp_largest, &search_left_bound_, &search_right_bound_);
&search_left_bound_,
&search_right_bound_);
} }
// Key falls out of current file's range // Key falls out of current file's range
if (cmp_smallest < 0 || cmp_largest > 0) { if (cmp_smallest < 0 || cmp_largest > 0) {
@ -846,22 +840,21 @@ Version::~Version() {
} }
int FindFile(const InternalKeyComparator& icmp, int FindFile(const InternalKeyComparator& icmp,
const LevelFilesBrief& file_level, const LevelFilesBrief& file_level, const Slice& key) {
const Slice& key) {
return FindFileInRange(icmp, file_level, key, 0, return FindFileInRange(icmp, file_level, key, 0,
static_cast<uint32_t>(file_level.num_files)); static_cast<uint32_t>(file_level.num_files));
} }
void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level, void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
const std::vector<FileMetaData*>& files, const std::vector<FileMetaData*>& files,
Arena* arena) { Arena* arena) {
assert(file_level); assert(file_level);
assert(arena); assert(arena);
size_t num = files.size(); size_t num = files.size();
file_level->num_files = num; file_level->num_files = num;
char* mem = arena->AllocateAligned(num * sizeof(FdWithKeyRange)); char* mem = arena->AllocateAligned(num * sizeof(FdWithKeyRange));
file_level->files = new (mem)FdWithKeyRange[num]; file_level->files = new (mem) FdWithKeyRange[num];
for (size_t i = 0; i < num; i++) { for (size_t i = 0; i < num; i++) {
Slice smallest_key = files[i]->smallest.Encode(); Slice smallest_key = files[i]->smallest.Encode();
@ -882,28 +875,27 @@ void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
} }
} }
static bool AfterFile(const Comparator* ucmp, static bool AfterFile(const Comparator* ucmp, const Slice* user_key,
const Slice* user_key, const FdWithKeyRange* f) { const FdWithKeyRange* f) {
// nullptr user_key occurs before all keys and is therefore never after *f // nullptr user_key occurs before all keys and is therefore never after *f
return (user_key != nullptr && return (user_key != nullptr &&
ucmp->CompareWithoutTimestamp(*user_key, ucmp->CompareWithoutTimestamp(*user_key,
ExtractUserKey(f->largest_key)) > 0); ExtractUserKey(f->largest_key)) > 0);
} }
static bool BeforeFile(const Comparator* ucmp, static bool BeforeFile(const Comparator* ucmp, const Slice* user_key,
const Slice* user_key, const FdWithKeyRange* f) { const FdWithKeyRange* f) {
// nullptr user_key occurs after all keys and is therefore never before *f // nullptr user_key occurs after all keys and is therefore never before *f
return (user_key != nullptr && return (user_key != nullptr &&
ucmp->CompareWithoutTimestamp(*user_key, ucmp->CompareWithoutTimestamp(*user_key,
ExtractUserKey(f->smallest_key)) < 0); ExtractUserKey(f->smallest_key)) < 0);
} }
bool SomeFileOverlapsRange( bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
const InternalKeyComparator& icmp, bool disjoint_sorted_files,
bool disjoint_sorted_files, const LevelFilesBrief& file_level,
const LevelFilesBrief& file_level, const Slice* smallest_user_key,
const Slice* smallest_user_key, const Slice* largest_user_key) {
const Slice* largest_user_key) {
const Comparator* ucmp = icmp.user_comparator(); const Comparator* ucmp = icmp.user_comparator();
if (!disjoint_sorted_files) { if (!disjoint_sorted_files) {
// Need to check against all files // Need to check against all files
@ -1025,9 +1017,7 @@ class LevelIterator final : public InternalIterator {
return file_iter_.iter() ? file_iter_.status() : Status::OK(); return file_iter_.iter() ? file_iter_.status() : Status::OK();
} }
bool PrepareValue() override { bool PrepareValue() override { return file_iter_.PrepareValue(); }
return file_iter_.PrepareValue();
}
inline bool MayBeOutOfLowerBound() override { inline bool MayBeOutOfLowerBound() override {
assert(Valid()); assert(Valid());
@ -1561,9 +1551,8 @@ Status Version::GetTableProperties(std::shared_ptr<const TableProperties>* tp,
if (fname != nullptr) { if (fname != nullptr) {
file_name = *fname; file_name = *fname;
} else { } else {
file_name = file_name = TableFileName(ioptions->cf_paths, file_meta->fd.GetNumber(),
TableFileName(ioptions->cf_paths, file_meta->fd.GetNumber(), file_meta->fd.GetPathId());
file_meta->fd.GetPathId());
} }
s = ioptions->fs->NewRandomAccessFile(file_name, file_options_, &file, s = ioptions->fs->NewRandomAccessFile(file_name, file_options_, &file,
nullptr); nullptr);
@ -1690,8 +1679,8 @@ Status Version::GetPropertiesOfTablesInRange(
false); false);
for (const auto& file_meta : files) { for (const auto& file_meta : files) {
auto fname = auto fname =
TableFileName(cfd_->ioptions()->cf_paths, TableFileName(cfd_->ioptions()->cf_paths, file_meta->fd.GetNumber(),
file_meta->fd.GetNumber(), file_meta->fd.GetPathId()); file_meta->fd.GetPathId());
if (props->count(fname) == 0) { if (props->count(fname) == 0) {
// 1. If the table is already present in table cache, load table // 1. If the table is already present in table cache, load table
// properties from there. // properties from there.
@ -1788,8 +1777,7 @@ void Version::GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta) {
files.back().num_deletions = file->num_deletions; files.back().num_deletions = file->num_deletions;
level_size += file->fd.GetFileSize(); level_size += file->fd.GetFileSize();
} }
cf_meta->levels.emplace_back( cf_meta->levels.emplace_back(level, level_size, std::move(files));
level, level_size, std::move(files));
cf_meta->size += level_size; cf_meta->size += level_size;
} }
for (const auto& meta : vstorage->GetBlobFiles()) { for (const auto& meta : vstorage->GetBlobFiles()) {
@ -1880,10 +1868,8 @@ uint64_t VersionStorageInfo::GetEstimatedActiveKeys() const {
if (current_num_samples_ < file_count) { if (current_num_samples_ < file_count) {
// casting to avoid overflowing // casting to avoid overflowing
return return static_cast<uint64_t>(
static_cast<uint64_t>( (est * static_cast<double>(file_count) / current_num_samples_));
(est * static_cast<double>(file_count) / current_num_samples_)
);
} else { } else {
return est; return est;
} }
@ -2020,8 +2006,8 @@ Status Version::OverlapWithLevelIterator(const ReadOptions& read_options,
/*smallest_compaction_key=*/nullptr, /*smallest_compaction_key=*/nullptr,
/*largest_compaction_key=*/nullptr, /*largest_compaction_key=*/nullptr,
/*allow_unprepared_value=*/false)); /*allow_unprepared_value=*/false));
status = OverlapWithIterator( status = OverlapWithIterator(ucmp, smallest_user_key, largest_user_key,
ucmp, smallest_user_key, largest_user_key, iter.get(), overlap); iter.get(), overlap);
if (!status.ok() || *overlap) { if (!status.ok() || *overlap) {
break; break;
} }
@ -2035,8 +2021,8 @@ Status Version::OverlapWithLevelIterator(const ReadOptions& read_options,
cfd_->internal_stats()->GetFileReadHist(level), cfd_->internal_stats()->GetFileReadHist(level),
TableReaderCaller::kUserIterator, IsFilterSkipped(level), level, TableReaderCaller::kUserIterator, IsFilterSkipped(level), level,
&range_del_agg)); &range_del_agg));
status = OverlapWithIterator( status = OverlapWithIterator(ucmp, smallest_user_key, largest_user_key,
ucmp, smallest_user_key, largest_user_key, iter.get(), overlap); iter.get(), overlap);
} }
if (status.ok() && *overlap == false && if (status.ok() && *overlap == false &&
@ -2396,7 +2382,7 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
return; return;
} }
if (!merge_operator_) { if (!merge_operator_) {
*status = Status::InvalidArgument( *status = Status::InvalidArgument(
"merge_operator is not properly initialized."); "merge_operator is not properly initialized.");
return; return;
} }
@ -2418,7 +2404,7 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
if (key_exists != nullptr) { if (key_exists != nullptr) {
*key_exists = false; *key_exists = false;
} }
*status = Status::NotFound(); // Use an empty error message for speed *status = Status::NotFound(); // Use an empty error message for speed
} }
} }
@ -2905,8 +2891,8 @@ bool Version::IsFilterSkipped(int level, bool is_file_last_in_level) {
void VersionStorageInfo::GenerateLevelFilesBrief() { void VersionStorageInfo::GenerateLevelFilesBrief() {
level_files_brief_.resize(num_non_empty_levels_); level_files_brief_.resize(num_non_empty_levels_);
for (int level = 0; level < num_non_empty_levels_; level++) { for (int level = 0; level < num_non_empty_levels_; level++) {
DoGenerateLevelFilesBrief( DoGenerateLevelFilesBrief(&level_files_brief_[level], files_[level],
&level_files_brief_[level], files_[level], &arena_); &arena_);
} }
} }
@ -2940,8 +2926,7 @@ void Version::PrepareAppend(const MutableCFOptions& mutable_cf_options,
} }
bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) { bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) {
if (file_meta->init_stats_from_file || if (file_meta->init_stats_from_file || file_meta->compensated_file_size > 0) {
file_meta->compensated_file_size > 0) {
return false; return false;
} }
std::shared_ptr<const TableProperties> tp; std::shared_ptr<const TableProperties> tp;
@ -3608,9 +3593,9 @@ struct Fsize {
// In normal mode: descending size // In normal mode: descending size
bool CompareCompensatedSizeDescending(const Fsize& first, const Fsize& second) { bool CompareCompensatedSizeDescending(const Fsize& first, const Fsize& second) {
return (first.file->compensated_file_size > return (first.file->compensated_file_size >
second.file->compensated_file_size); second.file->compensated_file_size);
} }
} // anonymous namespace } // anonymous namespace
void VersionStorageInfo::AddFile(int level, FileMetaData* f) { void VersionStorageInfo::AddFile(int level, FileMetaData* f) {
auto& level_files = files_[level]; auto& level_files = files_[level];
@ -3806,7 +3791,7 @@ void SortFileByRoundRobin(const InternalKeyComparator& icmp,
} }
} }
} }
} // namespace } // anonymous namespace
void VersionStorageInfo::UpdateFilesByCompactionPri( void VersionStorageInfo::UpdateFilesByCompactionPri(
const ImmutableOptions& ioptions, const MutableCFOptions& options) { const ImmutableOptions& ioptions, const MutableCFOptions& options) {
@ -3978,9 +3963,7 @@ void VersionStorageInfo::ComputeBottommostFilesMarkedForCompaction() {
} }
} }
void Version::Ref() { void Version::Ref() { ++refs_; }
++refs_;
}
bool Version::Unref() { bool Version::Unref() {
assert(refs_ >= 1); assert(refs_ >= 1);
@ -4112,9 +4095,8 @@ void VersionStorageInfo::GetCleanInputsWithinInterval(
return; return;
} }
GetOverlappingInputsRangeBinarySearch(level, begin, end, inputs, GetOverlappingInputsRangeBinarySearch(level, begin, end, inputs, hint_index,
hint_index, file_index, file_index, true /* within_interval */);
true /* within_interval */);
} }
// Store in "*inputs" all files in "level" that overlap [begin,end] // Store in "*inputs" all files in "level" that overlap [begin,end]
@ -4277,8 +4259,7 @@ const char* VersionStorageInfo::LevelFileSummary(FileSummaryStorage* scratch,
"#%" PRIu64 "(seq=%" PRIu64 ",sz=%s,%d) ", "#%" PRIu64 "(seq=%" PRIu64 ",sz=%s,%d) ",
f->fd.GetNumber(), f->fd.smallest_seqno, sztxt, f->fd.GetNumber(), f->fd.smallest_seqno, sztxt,
static_cast<int>(f->being_compacted)); static_cast<int>(f->being_compacted));
if (ret < 0 || ret >= sz) if (ret < 0 || ret >= sz) break;
break;
len += ret; len += ret;
} }
// overwrite the last space (only if files_[level].size() is non-zero) // overwrite the last space (only if files_[level].size() is non-zero)
@ -4456,13 +4437,13 @@ uint64_t VersionStorageInfo::EstimateLiveDataSize() const {
// no potential overlap, we can safely insert the rest of this level // no potential overlap, we can safely insert the rest of this level
// (if the level is not 0) into the map without checking again because // (if the level is not 0) into the map without checking again because
// the elements in the level are sorted and non-overlapping. // the elements in the level are sorted and non-overlapping.
auto lb = (found_end && l != 0) ? auto lb = (found_end && l != 0) ? ranges.end()
ranges.end() : ranges.lower_bound(&file->smallest); : ranges.lower_bound(&file->smallest);
found_end = (lb == ranges.end()); found_end = (lb == ranges.end());
if (found_end || internal_comparator_->Compare( if (found_end || internal_comparator_->Compare(
file->largest, (*lb).second->smallest) < 0) { file->largest, (*lb).second->smallest) < 0) {
ranges.emplace_hint(lb, &file->largest, file); ranges.emplace_hint(lb, &file->largest, file);
size += file->fd.file_size; size += file->fd.file_size;
} }
} }
} }
@ -5674,7 +5655,7 @@ std::string ManifestPicker::GetNextManifest(uint64_t* number,
} }
return ret; return ret;
} }
} // namespace } // anonymous namespace
Status VersionSet::TryRecover( Status VersionSet::TryRecover(
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only, const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
@ -5876,7 +5857,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
} }
} }
delete[] vstorage -> files_; delete[] vstorage->files_;
vstorage->files_ = new_files_list; vstorage->files_ = new_files_list;
vstorage->num_levels_ = new_levels; vstorage->num_levels_ = new_levels;
vstorage->ResizeCompactCursors(new_levels); vstorage->ResizeCompactCursors(new_levels);
@ -5885,9 +5866,9 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
VersionEdit ve; VersionEdit ve;
InstrumentedMutex dummy_mutex; InstrumentedMutex dummy_mutex;
InstrumentedMutexLock l(&dummy_mutex); InstrumentedMutexLock l(&dummy_mutex);
return versions.LogAndApply( return versions.LogAndApply(versions.GetColumnFamilySet()->GetDefault(),
versions.GetColumnFamilySet()->GetDefault(), mutable_cf_options, &ve, &dummy_mutex, nullptr,
mutable_cf_options, &ve, &dummy_mutex, nullptr, true); true);
} }
// Get the checksum information including the checksum and checksum function // Get the checksum information including the checksum and checksum function
@ -5973,9 +5954,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
std::unique_ptr<FSSequentialFile> file; std::unique_ptr<FSSequentialFile> file;
const std::shared_ptr<FileSystem>& fs = options.env->GetFileSystem(); const std::shared_ptr<FileSystem>& fs = options.env->GetFileSystem();
s = fs->NewSequentialFile( s = fs->NewSequentialFile(
dscname, dscname, fs->OptimizeForManifestRead(file_options_), &file, nullptr);
fs->OptimizeForManifestRead(file_options_), &file,
nullptr);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -6078,8 +6057,8 @@ Status VersionSet::WriteCurrentStateToManifest(
cfd->internal_comparator().user_comparator()->Name()); cfd->internal_comparator().user_comparator()->Name());
std::string record; std::string record;
if (!edit.EncodeTo(&record)) { if (!edit.EncodeTo(&record)) {
return Status::Corruption( return Status::Corruption("Unable to Encode VersionEdit:" +
"Unable to Encode VersionEdit:" + edit.DebugString(true)); edit.DebugString(true));
} }
io_s = log->AddRecord(record); io_s = log->AddRecord(record);
if (!io_s.ok()) { if (!io_s.ok()) {
@ -6137,9 +6116,10 @@ Status VersionSet::WriteCurrentStateToManifest(
edit.SetLogNumber(log_number); edit.SetLogNumber(log_number);
if (cfd->GetID() == 0) { if (cfd->GetID() == 0) {
// min_log_number_to_keep is for the whole db, not for specific column family. // min_log_number_to_keep is for the whole db, not for specific column
// So it does not need to be set for every column family, just need to be set once. // family. So it does not need to be set for every column family, just
// Since default CF can never be dropped, we set the min_log to the default CF here. // need to be set once. Since default CF can never be dropped, we set
// the min_log to the default CF here.
uint64_t min_log = min_log_number_to_keep(); uint64_t min_log = min_log_number_to_keep();
if (min_log != 0) { if (min_log != 0) {
edit.SetMinLogNumberToKeep(min_log); edit.SetMinLogNumberToKeep(min_log);
@ -6155,8 +6135,8 @@ Status VersionSet::WriteCurrentStateToManifest(
std::string record; std::string record;
if (!edit.EncodeTo(&record)) { if (!edit.EncodeTo(&record)) {
return Status::Corruption( return Status::Corruption("Unable to Encode VersionEdit:" +
"Unable to Encode VersionEdit:" + edit.DebugString(true)); edit.DebugString(true));
} }
io_s = log->AddRecord(record); io_s = log->AddRecord(record);
if (!io_s.ok()) { if (!io_s.ok()) {
@ -6479,7 +6459,7 @@ InternalIterator* VersionSet::MakeInputIterator(
const size_t space = (c->level() == 0 ? c->input_levels(0)->num_files + const size_t space = (c->level() == 0 ? c->input_levels(0)->num_files +
c->num_input_levels() - 1 c->num_input_levels() - 1
: c->num_input_levels()); : c->num_input_levels());
InternalIterator** list = new InternalIterator* [space]; InternalIterator** list = new InternalIterator*[space];
size_t num = 0; size_t num = 0;
for (size_t which = 0; which < c->num_input_levels(); which++) { for (size_t which = 0; which < c->num_input_levels(); which++) {
if (c->input_levels(which)->num_files != 0) { if (c->input_levels(which)->num_files != 0) {
@ -6588,8 +6568,8 @@ void VersionSet::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
filemetadata.largestkey = file->largest.user_key().ToString(); filemetadata.largestkey = file->largest.user_key().ToString();
filemetadata.smallest_seqno = file->fd.smallest_seqno; filemetadata.smallest_seqno = file->fd.smallest_seqno;
filemetadata.largest_seqno = file->fd.largest_seqno; filemetadata.largest_seqno = file->fd.largest_seqno;
filemetadata.num_reads_sampled = file->stats.num_reads_sampled.load( filemetadata.num_reads_sampled =
std::memory_order_relaxed); file->stats.num_reads_sampled.load(std::memory_order_relaxed);
filemetadata.being_compacted = file->being_compacted; filemetadata.being_compacted = file->being_compacted;
filemetadata.num_entries = file->num_entries; filemetadata.num_entries = file->num_entries;
filemetadata.num_deletions = file->num_deletions; filemetadata.num_deletions = file->num_deletions;

@ -230,9 +230,7 @@ class VersionStorageInfo {
double blob_garbage_collection_age_cutoff, double blob_garbage_collection_age_cutoff,
double blob_garbage_collection_force_threshold); double blob_garbage_collection_force_threshold);
bool level0_non_overlapping() const { bool level0_non_overlapping() const { return level0_non_overlapping_; }
return level0_non_overlapping_;
}
// Updates the oldest snapshot and related internal state, like the bottommost // Updates the oldest snapshot and related internal state, like the bottommost
// files marked for compaction. // files marked for compaction.
@ -814,8 +812,8 @@ class Version {
Status OverlapWithLevelIterator(const ReadOptions&, const FileOptions&, Status OverlapWithLevelIterator(const ReadOptions&, const FileOptions&,
const Slice& smallest_user_key, const Slice& smallest_user_key,
const Slice& largest_user_key, const Slice& largest_user_key, int level,
int level, bool* overlap); bool* overlap);
// Lookup the value for key or get all merge operands for key. // Lookup the value for key or get all merge operands for key.
// If do_merge = true (default) then lookup value for key. // If do_merge = true (default) then lookup value for key.
@ -1032,10 +1030,10 @@ class Version {
const MergeOperator* merge_operator_; const MergeOperator* merge_operator_;
VersionStorageInfo storage_info_; VersionStorageInfo storage_info_;
VersionSet* vset_; // VersionSet to which this Version belongs VersionSet* vset_; // VersionSet to which this Version belongs
Version* next_; // Next version in linked list Version* next_; // Next version in linked list
Version* prev_; // Previous version in linked list Version* prev_; // Previous version in linked list
int refs_; // Number of live refs to this version int refs_; // Number of live refs to this version
const FileOptions file_options_; const FileOptions file_options_;
const MutableCFOptions mutable_cf_options_; const MutableCFOptions mutable_cf_options_;
// Cached value to avoid recomputing it on every read. // Cached value to avoid recomputing it on every read.
@ -1398,7 +1396,7 @@ class VersionSet {
FileMetaData** metadata, ColumnFamilyData** cfd); FileMetaData** metadata, ColumnFamilyData** cfd);
// This function doesn't support leveldb SST filenames // This function doesn't support leveldb SST filenames
void GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata); void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) { void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) {
assert(table_cache_); assert(table_cache_);

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save