|
|
|
|
# This cmake build is for Windows 64-bit only.
|
|
|
|
|
#
|
|
|
|
|
# Prerequisites:
|
|
|
|
|
# You must have Visual Studio 2013 Update 4 installed. Start the Developer Command Prompt window that is a part of Visual Studio installation.
|
|
|
|
|
# Run the build commands from within the Developer Command Prompt window to have paths to the compiler and runtime libraries set.
|
|
|
|
|
# You must have git.exe in your %PATH% environment variable.
|
|
|
|
|
#
|
|
|
|
|
# To build Rocksdb for Windows is as easy as 1-2-3-4-5:
|
|
|
|
|
#
|
|
|
|
|
# 1. Update paths to third-party libraries in thirdparty.inc file
|
|
|
|
|
# 2. Create a new directory for build artifacts
|
|
|
|
|
# mkdir build
|
|
|
|
|
# cd build
|
|
|
|
|
# 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
|
|
|
|
|
# See thirdparty.inc for more information.
|
|
|
|
|
# sample command: cmake -G "Visual Studio 12 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
|
|
|
|
|
# OR for VS Studio 15 cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
|
|
|
|
|
# 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
|
|
|
|
|
# or simply /m ot use all avail cores)
|
|
|
|
|
# msbuild rocksdb.sln
|
|
|
|
|
#
|
|
|
|
|
# rocksdb.sln build features exclusions of test only code in Release. If you build ALL_BUILD then everything
|
|
|
|
|
# will be attempted but test only code does not build in Release mode.
|
|
|
|
|
#
|
|
|
|
|
# 5. And release mode (/m[:<N>] is also supported)
|
|
|
|
|
# msbuild rocksdb.sln /p:Configuration=Release
|
|
|
|
|
#
|
|
|
|
|
|
|
|
|
|
cmake_minimum_required(VERSION 2.6)
|
|
|
|
|
project(rocksdb)
|
|
|
|
|
|
|
|
|
|
include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
|
|
|
|
|
execute_process(COMMAND powershell -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
|
|
|
|
|
execute_process(COMMAND powershell -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
|
|
|
|
|
string(REGEX REPLACE "(..)_(..)_..(..).*" "\\1/\\2/\\3" DATE "${DATE}")
|
|
|
|
|
string(REGEX REPLACE "(..):(.....).*" " \\1:\\2" TIME "${TIME}")
|
|
|
|
|
string(CONCAT GIT_DATE_TIME ${DATE} ${TIME})
|
|
|
|
|
|
|
|
|
|
find_package(Git)
|
|
|
|
|
|
|
|
|
|
if (GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
|
|
|
|
|
execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
|
|
|
|
|
else()
|
|
|
|
|
set(GIT_SHA 0)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
|
|
|
|
|
|
|
|
|
|
set(BUILD_VERSION_CC ${CMAKE_CURRENT_SOURCE_DIR}/util/build_version.cc)
|
|
|
|
|
|
|
|
|
|
add_custom_command(OUTPUT ${BUILD_VERSION_CC}
|
|
|
|
|
COMMAND echo "#include \"build_version.h\"" > ${BUILD_VERSION_CC}
|
|
|
|
|
COMMAND echo "const char* rocksdb_build_git_sha = \"rocksdb_build_git_sha:${GIT_SHA}\";" >> ${BUILD_VERSION_CC}
|
|
|
|
|
COMMAND echo "const char* rocksdb_build_git_datetime = \"rocksdb_build_git_datetime:${GIT_DATE_TIME}\";" >> ${BUILD_VERSION_CC}
|
|
|
|
|
COMMAND echo const char* rocksdb_build_compile_date = __DATE__\; >> ${BUILD_VERSION_CC}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
add_custom_target(GenerateBuildVersion DEPENDS ${BUILD_VERSION_CC})
|
|
|
|
|
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
|
|
|
|
|
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /WX /wd4127 /wd4800 /wd4996 /wd4351")
|
|
|
|
|
|
|
|
|
|
# Used to run CI build and tests so we can run faster
|
|
|
|
|
set(OPTIMIZE_DEBUG_DEFAULT 0) # Debug build is unoptimized by default use -DOPTDBG=1 to optimize
|
|
|
|
|
|
|
|
|
|
if(DEFINED OPTDBG)
|
|
|
|
|
set(OPTIMIZE_DEBUG ${OPTDBG})
|
|
|
|
|
else()
|
|
|
|
|
set(OPTIMIZE_DEBUG ${OPTIMIZE_DEBUG_DEFAULT})
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if((${OPTIMIZE_DEBUG} EQUAL 1))
|
|
|
|
|
message(STATUS "Debug optimization is enabled")
|
|
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /MDd")
|
|
|
|
|
else()
|
|
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd")
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /MD")
|
|
|
|
|
|
|
|
|
|
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
|
|
|
|
|
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
|
|
|
|
|
|
|
|
|
|
add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)
|
|
|
|
|
|
|
|
|
|
include_directories(${PROJECT_SOURCE_DIR})
|
|
|
|
|
include_directories(${PROJECT_SOURCE_DIR}/include)
|
|
|
|
|
include_directories(${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
|
|
|
|
|
|
|
|
|
|
set(ROCKSDB_LIBS rocksdblib${ARTIFACT_SUFFIX})
|
|
|
|
|
set(THIRDPARTY_LIBS ${THIRDPARTY_LIBS} gtest)
|
|
|
|
|
set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
|
|
|
|
|
|
|
|
|
|
set(LIBS ${ROCKSDB_LIBS} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
|
|
|
|
|
|
|
|
|
add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest)
|
|
|
|
|
|
|
|
|
|
# Main library source code
|
|
|
|
|
set(SOURCES
|
|
|
|
|
db/auto_roll_logger.cc
|
|
|
|
|
db/builder.cc
|
|
|
|
|
db/c.cc
|
|
|
|
|
db/column_family.cc
|
|
|
|
|
db/compacted_db_impl.cc
|
|
|
|
|
db/compaction.cc
|
|
|
|
|
db/compaction_iterator.cc
|
|
|
|
|
db/compaction_job.cc
|
|
|
|
|
db/compaction_picker.cc
|
|
|
|
|
db/convenience.cc
|
|
|
|
|
db/dbformat.cc
|
|
|
|
|
db/db_filesnapshot.cc
|
|
|
|
|
db/db_impl.cc
|
|
|
|
|
db/db_impl_debug.cc
|
|
|
|
|
db/db_impl_experimental.cc
|
|
|
|
|
db/db_impl_add_file.cc
|
|
|
|
|
db/db_impl_readonly.cc
|
|
|
|
|
db/db_info_dumper.cc
|
|
|
|
|
db/db_iter.cc
|
|
|
|
|
db/event_helpers.cc
|
|
|
|
|
db/experimental.cc
|
|
|
|
|
db/filename.cc
|
|
|
|
|
db/file_indexer.cc
|
|
|
|
|
db/flush_job.cc
|
|
|
|
|
db/flush_scheduler.cc
|
|
|
|
|
db/forward_iterator.cc
|
|
|
|
|
db/internal_stats.cc
|
|
|
|
|
db/log_reader.cc
|
|
|
|
|
db/log_writer.cc
|
|
|
|
|
db/managed_iterator.cc
|
|
|
|
|
db/memtable.cc
|
|
|
|
|
db/memtable_allocator.cc
|
|
|
|
|
db/memtable_list.cc
|
|
|
|
|
db/merge_helper.cc
|
|
|
|
|
db/merge_operator.cc
|
|
|
|
|
db/repair.cc
|
|
|
|
|
db/snapshot_impl.cc
|
|
|
|
|
db/table_cache.cc
|
|
|
|
|
db/table_properties_collector.cc
|
|
|
|
|
db/transaction_log_impl.cc
|
|
|
|
|
db/version_builder.cc
|
|
|
|
|
db/version_edit.cc
|
|
|
|
|
db/version_set.cc
|
|
|
|
|
db/wal_manager.cc
|
|
|
|
|
db/write_batch.cc
|
|
|
|
|
db/write_batch_base.cc
|
|
|
|
|
db/write_controller.cc
|
|
|
|
|
db/write_thread.cc
|
|
|
|
|
db/xfunc_test_points.cc
|
|
|
|
|
memtable/hash_cuckoo_rep.cc
|
|
|
|
|
memtable/hash_linklist_rep.cc
|
|
|
|
|
memtable/hash_skiplist_rep.cc
|
|
|
|
|
memtable/skiplistrep.cc
|
|
|
|
|
memtable/vectorrep.cc
|
|
|
|
|
port/stack_trace.cc
|
|
|
|
|
port/win/io_win.cc
|
|
|
|
|
port/win/env_win.cc
|
|
|
|
|
port/win/env_default.cc
|
|
|
|
|
port/win/port_win.cc
|
|
|
|
|
port/win/win_logger.cc
|
|
|
|
|
port/win/xpress_win.cc
|
|
|
|
|
table/adaptive_table_factory.cc
|
|
|
|
|
table/block.cc
|
|
|
|
|
table/block_based_filter_block.cc
|
|
|
|
|
table/block_based_table_builder.cc
|
|
|
|
|
table/block_based_table_factory.cc
|
|
|
|
|
table/block_based_table_reader.cc
|
|
|
|
|
table/block_builder.cc
|
|
|
|
|
table/block_prefix_index.cc
|
|
|
|
|
table/bloom_block.cc
|
|
|
|
|
table/cuckoo_table_builder.cc
|
|
|
|
|
table/cuckoo_table_factory.cc
|
|
|
|
|
table/cuckoo_table_reader.cc
|
|
|
|
|
table/flush_block_policy.cc
|
|
|
|
|
table/format.cc
|
|
|
|
|
table/full_filter_block.cc
|
|
|
|
|
table/get_context.cc
|
|
|
|
|
table/iterator.cc
|
|
|
|
|
table/merger.cc
|
|
|
|
|
table/sst_file_writer.cc
|
|
|
|
|
table/meta_blocks.cc
|
|
|
|
|
table/plain_table_builder.cc
|
|
|
|
|
table/plain_table_factory.cc
|
|
|
|
|
table/plain_table_index.cc
|
|
|
|
|
table/plain_table_key_coding.cc
|
|
|
|
|
table/plain_table_reader.cc
|
|
|
|
|
table/persistent_cache_helper.cc
|
|
|
|
|
table/table_properties.cc
|
|
|
|
|
table/two_level_iterator.cc
|
|
|
|
|
tools/sst_dump_tool.cc
|
|
|
|
|
tools/db_bench_tool.cc
|
|
|
|
|
tools/dump/db_dump_tool.cc
|
|
|
|
|
util/arena.cc
|
|
|
|
|
util/bloom.cc
|
|
|
|
|
util/build_version.cc
|
|
|
|
|
util/coding.cc
|
|
|
|
|
util/compaction_job_stats_impl.cc
|
|
|
|
|
util/comparator.cc
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
9 years ago
|
|
|
util/concurrent_arena.cc
|
|
|
|
|
util/crc32c.cc
|
|
|
|
|
util/delete_scheduler.cc
|
|
|
|
|
util/dynamic_bloom.cc
|
|
|
|
|
util/env.cc
|
|
|
|
|
util/env_chroot.cc
|
|
|
|
|
util/env_hdfs.cc
|
|
|
|
|
util/event_logger.cc
|
|
|
|
|
util/file_util.cc
|
|
|
|
|
util/file_reader_writer.cc
|
|
|
|
|
util/sst_file_manager_impl.cc
|
|
|
|
|
util/filter_policy.cc
|
|
|
|
|
util/hash.cc
|
|
|
|
|
util/histogram.cc
|
|
|
|
|
util/histogram_windowing.cc
|
|
|
|
|
util/instrumented_mutex.cc
|
|
|
|
|
util/iostats_context.cc
|
|
|
|
|
util/lru_cache.cc
|
|
|
|
|
tools/ldb_cmd.cc
|
|
|
|
|
tools/ldb_tool.cc
|
|
|
|
|
util/logging.cc
|
|
|
|
|
util/log_buffer.cc
|
|
|
|
|
util/memenv.cc
|
|
|
|
|
util/murmurhash.cc
|
|
|
|
|
util/mutable_cf_options.cc
|
|
|
|
|
util/options.cc
|
|
|
|
|
util/options_helper.cc
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
9 years ago
|
|
|
util/options_parser.cc
|
|
|
|
|
util/options_sanity_check.cc
|
|
|
|
|
util/perf_context.cc
|
|
|
|
|
util/perf_level.cc
|
|
|
|
|
util/random.cc
|
|
|
|
|
util/rate_limiter.cc
|
|
|
|
|
util/sharded_cache.cc
|
|
|
|
|
util/slice.cc
|
|
|
|
|
util/statistics.cc
|
|
|
|
|
util/status.cc
|
|
|
|
|
util/status_message.cc
|
|
|
|
|
util/string_util.cc
|
|
|
|
|
util/sync_point.cc
|
|
|
|
|
util/testharness.cc
|
|
|
|
|
util/testutil.cc
|
|
|
|
|
util/thread_local.cc
|
|
|
|
|
util/threadpool.cc
|
|
|
|
|
util/thread_status_impl.cc
|
|
|
|
|
util/thread_status_updater.cc
|
|
|
|
|
util/thread_status_util.cc
|
|
|
|
|
util/thread_status_util_debug.cc
|
|
|
|
|
util/transaction_test_util.cc
|
|
|
|
|
util/xfunc.cc
|
|
|
|
|
util/xxhash.cc
|
|
|
|
|
utilities/backupable/backupable_db.cc
|
|
|
|
|
utilities/checkpoint/checkpoint.cc
|
|
|
|
|
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
|
|
|
|
|
utilities/document/document_db.cc
|
|
|
|
|
utilities/document/json_document.cc
|
|
|
|
|
utilities/document/json_document_builder.cc
|
|
|
|
|
utilities/env_mirror.cc
|
|
|
|
|
utilities/env_registry.cc
|
|
|
|
|
utilities/flashcache/flashcache.cc
|
|
|
|
|
utilities/geodb/geodb_impl.cc
|
|
|
|
|
utilities/leveldb_options/leveldb_options.cc
|
|
|
|
|
utilities/memory/memory_util.cc
|
|
|
|
|
utilities/merge_operators/string_append/stringappend.cc
|
|
|
|
|
utilities/merge_operators/string_append/stringappend2.cc
|
|
|
|
|
utilities/merge_operators/put.cc
|
|
|
|
|
utilities/merge_operators/max.cc
|
|
|
|
|
utilities/merge_operators/uint64add.cc
|
Add OptionsUtil::LoadOptionsFromFile() API
Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
9 years ago
|
|
|
utilities/options/options_util.cc
|
|
|
|
|
utilities/persistent_cache/persistent_cache_tier.cc
|
|
|
|
|
utilities/persistent_cache/volatile_tier_impl.cc
|
|
|
|
|
utilities/redis/redis_lists.cc
|
|
|
|
|
utilities/simulator_cache/sim_cache.cc
|
|
|
|
|
utilities/spatialdb/spatial_db.cc
|
|
|
|
|
utilities/table_properties_collectors/compact_on_deletion_collector.cc
|
|
|
|
|
utilities/transactions/optimistic_transaction_impl.cc
|
|
|
|
|
utilities/transactions/optimistic_transaction_db_impl.cc
|
|
|
|
|
utilities/transactions/transaction_base.cc
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
10 years ago
|
|
|
utilities/transactions/transaction_impl.cc
|
|
|
|
|
utilities/transactions/transaction_db_impl.cc
|
|
|
|
|
utilities/transactions/transaction_db_mutex_impl.cc
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
10 years ago
|
|
|
utilities/transactions/transaction_lock_mgr.cc
|
|
|
|
|
utilities/transactions/transaction_util.cc
|
|
|
|
|
utilities/ttl/db_ttl_impl.cc
|
|
|
|
|
utilities/write_batch_with_index/write_batch_with_index.cc
|
|
|
|
|
utilities/write_batch_with_index/write_batch_with_index_internal.cc
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# For test util library that is build only in DEBUG mode
|
|
|
|
|
# and linked to tests. Add test only code that is not #ifdefed for Release here.
|
|
|
|
|
set(TESTUTIL_SOURCE
|
|
|
|
|
db/db_test_util.cc
|
|
|
|
|
table/mock_table.cc
|
|
|
|
|
util/mock_env.cc
|
[rocksdb] Recovery path sequence miscount fix
Summary:
Consider the following WAL with 4 batch entries prefixed with their sequence at time of memtable insert.
[1: BEGIN_PREPARE, PUT, PUT, PUT, PUT, END_PREPARE(a)]
[1: BEGIN_PREPARE, PUT, PUT, PUT, PUT, END_PREPARE(b)]
[4: COMMIT(a)]
[7: COMMIT(b)]
The first two batches do not consume any sequence numbers so are both prefixed with seq=1.
For 2pc commit, memtable insertion takes place before COMMIT batch is written to WAL.
We can see that sequence number consumption takes place between WAL entries giving us the seemingly sparse sequence prefix for WAL entries.
This is a valid WAL.
Because with 2PC markers one WriteBatch points to another batch containing its inserts a writebatch can consume more or less sequence numbers than the number of sequence consuming entries that it contains.
We can see that, given the entries in the WAL, 6 sequence ids were consumed. Yet on recovery the maximum sequence consumed would be 7 + 3 (the number of sequence numbers consumed by COMMIT(b))
So, now upon recovery we must track the actual consumption of sequence numbers.
In the provided scenario there will be no sequence gaps, but it is possible to produce a sequence gap. This should not be a problem though. correct?
Test Plan: provided test.
Reviewers: sdong
Subscribers: andrewkr, leveldb, dhruba, hermanlee4
Differential Revision: https://reviews.facebook.net/D57645
9 years ago
|
|
|
util/fault_injection_test_env.cc
|
|
|
|
|
util/thread_status_updater_debug.cc
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
add_library(rocksdblib${ARTIFACT_SUFFIX} ${SOURCES})
|
|
|
|
|
set_target_properties(rocksdblib${ARTIFACT_SUFFIX} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/rocksdblib${ARTIFACT_SUFFIX}.pdb")
|
|
|
|
|
add_dependencies(rocksdblib${ARTIFACT_SUFFIX} GenerateBuildVersion)
|
|
|
|
|
|
|
|
|
|
add_library(rocksdb${ARTIFACT_SUFFIX} SHARED ${SOURCES})
|
|
|
|
|
set_target_properties(rocksdb${ARTIFACT_SUFFIX} PROPERTIES COMPILE_FLAGS "-DROCKSDB_DLL -DROCKSDB_LIBRARY_EXPORTS /Fd${CMAKE_CFG_INTDIR}/rocksdb${ARTIFACT_SUFFIX}.pdb")
|
|
|
|
|
add_dependencies(rocksdb${ARTIFACT_SUFFIX} GenerateBuildVersion)
|
|
|
|
|
target_link_libraries(rocksdb${ARTIFACT_SUFFIX} ${LIBS})
|
|
|
|
|
|
|
|
|
|
if (DEFINED JNI)
|
|
|
|
|
if (${JNI} EQUAL 1)
|
|
|
|
|
message(STATUS "JNI library is enabled")
|
|
|
|
|
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
|
|
|
|
|
else()
|
|
|
|
|
message(STATUS "JNI library is disabled")
|
|
|
|
|
endif()
|
|
|
|
|
else()
|
|
|
|
|
message(STATUS "JNI library is disabled")
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
set(APPS
|
|
|
|
|
tools/db_bench.cc
|
|
|
|
|
db/memtablerep_bench.cc
|
|
|
|
|
table/table_reader_bench.cc
|
|
|
|
|
tools/db_stress.cc
|
Write stress test
Summary:
The goal of this diff is to create a simple stress test with focus on catching:
* bugs in compaction/flush processes, especially the ones that cause assertion errors
* bugs in the code that deletes obsolete files
There are two parts of the test:
* write_stress, a binary that writes to the database
* write_stress_runner.py, a script that invokes and kills write_stress
Here are some interesting parts of write_stress:
* Runs with very high concurrency of compactions and flushes (32 threads total) and tries to create a huge amount of small files
* The keys written to the database are not uniformly distributed -- there is a 3-character prefix that mutates occasionally (in prefix mutator thread), in such a way that the first character mutates slower than second, which mutates slower than third character. That way, the compaction stress tests some interesting compaction features like trivial moves and bottommost level calculation
* There is a thread that creates an iterator, holds it for couple of seconds and then iterates over all keys. This is supposed to test RocksDB's abilities to keep the files alive when there are references to them.
* Some writes trigger WAL sync. This is stress testing our WAL sync code.
* At the end of the run, we make sure that we didn't leak any of the sst files
write_stress_runner.py changes the mode in which we run write_stress and also kills and restarts it. There are some interesting characteristics:
* At the beginning we divide the full test runtime into smaller parts -- shorter runtimes (couple of seconds) and longer runtimes (100, 1000) seconds
* The first time we run write_stress, we destroy the old DB. Every next time during the test, we use the same DB.
* We can run in kill mode or clean-restart mode. Kill mode kills the write_stress violently.
* We can run in mode where delete_obsolete_files_with_fullscan is true or false
* We can run with low_open_files mode turned on or off. When it's turned on, we configure table cache to only hold a couple of files -- that way we need to reopen files every time we access them.
Another goal was to create a stress test without a lot of parameters. So tools/write_stress_runner.py should only take one parameter -- runtime_sec and it should figure out everything else on its own.
In a separate diff, I'll add this new test to our nightly legocastle runs.
Test Plan:
The goal of this test was to retroactively catch the following bugs: D33045, D48201, D46899, D42399. I failed to reproduce D48201, but all others have been caught!
When i reverted https://reviews.facebook.net/D33045:
./write_stress --runtime_sec=200 --low_open_files_mode=true
Iterator statuts not OK: IO error: /fast-rocksdb-tmp/rocksdb_test/write_stress/089166.sst: No such file or directory
When i reverted https://reviews.facebook.net/D42399:
python tools/write_stress_runner.py --runtime_sec=5000
Running write_stress, will kill after 5 seconds: ./write_stress --runtime_sec=-1
Running write_stress, will kill after 2 seconds: ./write_stress --runtime_sec=-1 --destroy_db=false --delete_obsolete_files_with_fullscan=true
Running write_stress, will kill after 7 seconds: ./write_stress --runtime_sec=-1 --destroy_db=false
Running write_stress, will kill after 5 seconds: ./write_stress --runtime_sec=-1 --destroy_db=false
Running write_stress, will kill after 8 seconds: ./write_stress --runtime_sec=-1 --destroy_db=false --low_open_files_mode=true
Write to DB failed: IO error: /fast-rocksdb-tmp/rocksdb_test/write_stress/019250.sst: No such file or directory
ERROR: write_stress died with exitcode=-6
When i reverted https://reviews.facebook.net/D46899:
python tools/write_stress_runner.py --runtime_sec=1000
runtime: 1000
Going to execute write stress for [3, 3, 100, 3, 2, 100, 1, 788]
Running write_stress for 3 seconds: ./write_stress --runtime_sec=3 --low_open_files_mode=true
Running write_stress for 3 seconds: ./write_stress --runtime_sec=3 --destroy_db=false --delete_obsolete_files_with_fullscan=true
Running write_stress, will kill after 100 seconds: ./write_stress --runtime_sec=-1 --destroy_db=false --delete_obsolete_files_with_fullscan=true
write_stress: db/db_impl.cc:2070: void rocksdb::DBImpl::MarkLogsSynced(uint64_t, bool, const rocksdb::Status&): Assertion `log.getting_synced' failed.
ERROR: write_stress died with exitcode=-6
Reviewers: IslamAbdelRahman, yhchiang, rven, kradhakrishnan, sdong, anthony
Reviewed By: anthony
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D49533
9 years ago
|
|
|
tools/write_stress.cc
|
|
|
|
|
tools/db_repl_stress.cc
|
|
|
|
|
tools/ldb.cc
|
|
|
|
|
tools/sst_dump.cc
|
|
|
|
|
tools/dump/rocksdb_dump.cc
|
|
|
|
|
tools/dump/rocksdb_undump.cc
|
|
|
|
|
util/cache_bench.cc
|
|
|
|
|
utilities/persistent_cache/hash_table_bench.cc
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
set(C_TESTS db/c_test.c)
|
|
|
|
|
|
|
|
|
|
set(TESTS
|
|
|
|
|
db/auto_roll_logger_test.cc
|
|
|
|
|
db/column_family_test.cc
|
|
|
|
|
db/compact_files_test.cc
|
|
|
|
|
db/compaction_iterator_test.cc
|
|
|
|
|
db/compaction_job_test.cc
|
|
|
|
|
db/compaction_job_stats_test.cc
|
|
|
|
|
db/compaction_picker_test.cc
|
|
|
|
|
db/comparator_db_test.cc
|
|
|
|
|
db/corruption_test.cc
|
|
|
|
|
db/cuckoo_table_db_test.cc
|
|
|
|
|
db/db_compaction_filter_test.cc
|
|
|
|
|
db/db_compaction_test.cc
|
|
|
|
|
db/db_dynamic_level_test.cc
|
|
|
|
|
db/db_inplace_update_test.cc
|
|
|
|
|
db/db_iter_test.cc
|
|
|
|
|
db/db_log_iter_test.cc
|
|
|
|
|
db/db_options_test.cc
|
|
|
|
|
db/db_properties_test.cc
|
|
|
|
|
db/db_table_properties_test.cc
|
|
|
|
|
db/db_tailing_iter_test.cc
|
|
|
|
|
db/db_test.cc
|
|
|
|
|
db/db_test2.cc
|
|
|
|
|
db/db_block_cache_test.cc
|
|
|
|
|
db/db_bloom_filter_test.cc
|
Fix flush not being commit while writing manifest
Summary:
Fix flush not being commit while writing manifest, which is a recent bug introduced by D60075.
The issue:
# Options.max_background_flushes > 1
# Background thread A pick up a flush job, flush, then commit to manifest. (Note that mutex is released before writing manifest.)
# Background thread B pick up another flush job, flush. When it gets to `MemTableList::InstallMemtableFlushResults`, it notices another thread is commiting, so it quit.
# After the first commit, thread A doesn't double check if there are more flush result need to commit, leaving the second flush uncommitted.
Test Plan: run the test. Also verify the new test hit deadlock without the fix.
Reviewers: sdong, igor, lightmark
Reviewed By: lightmark
Subscribers: andrewkr, omegaga, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D60969
8 years ago
|
|
|
db/db_flush_test.cc
|
|
|
|
|
db/db_iterator_test.cc
|
|
|
|
|
db/db_sst_test.cc
|
|
|
|
|
db/db_universal_compaction_test.cc
|
|
|
|
|
db/db_wal_test.cc
|
|
|
|
|
db/dbformat_test.cc
|
|
|
|
|
db/deletefile_test.cc
|
|
|
|
|
db/fault_injection_test.cc
|
|
|
|
|
db/file_indexer_test.cc
|
|
|
|
|
db/filename_test.cc
|
|
|
|
|
db/flush_job_test.cc
|
|
|
|
|
db/inlineskiplist_test.cc
|
|
|
|
|
db/listener_test.cc
|
|
|
|
|
db/log_test.cc
|
|
|
|
|
db/manual_compaction_test.cc
|
|
|
|
|
db/memtable_list_test.cc
|
|
|
|
|
db/merge_test.cc
|
|
|
|
|
db/merge_helper_test.cc
|
|
|
|
|
db/options_file_test.cc
|
|
|
|
|
db/perf_context_test.cc
|
|
|
|
|
db/plain_table_db_test.cc
|
|
|
|
|
db/prefix_test.cc
|
|
|
|
|
db/repair_test.cc
|
|
|
|
|
db/skiplist_test.cc
|
|
|
|
|
db/table_properties_collector_test.cc
|
|
|
|
|
db/version_builder_test.cc
|
|
|
|
|
db/version_edit_test.cc
|
|
|
|
|
db/version_set_test.cc
|
|
|
|
|
db/wal_manager_test.cc
|
|
|
|
|
db/write_batch_test.cc
|
|
|
|
|
db/write_callback_test.cc
|
|
|
|
|
db/write_controller_test.cc
|
|
|
|
|
db/db_io_failure_test.cc
|
|
|
|
|
table/block_based_filter_block_test.cc
|
|
|
|
|
table/block_test.cc
|
|
|
|
|
table/cuckoo_table_builder_test.cc
|
|
|
|
|
table/cuckoo_table_reader_test.cc
|
|
|
|
|
table/full_filter_block_test.cc
|
|
|
|
|
table/merger_test.cc
|
|
|
|
|
table/table_test.cc
|
|
|
|
|
tools/db_sanity_test.cc
|
|
|
|
|
tools/ldb_cmd_test.cc
|
|
|
|
|
tools/reduce_levels_test.cc
|
|
|
|
|
tools/sst_dump_test.cc
|
|
|
|
|
util/arena_test.cc
|
|
|
|
|
util/autovector_test.cc
|
|
|
|
|
util/bloom_test.cc
|
|
|
|
|
util/cache_test.cc
|
|
|
|
|
util/coding_test.cc
|
|
|
|
|
util/crc32c_test.cc
|
|
|
|
|
util/delete_scheduler_test.cc
|
|
|
|
|
util/dynamic_bloom_test.cc
|
|
|
|
|
util/env_basic_test.cc
|
|
|
|
|
util/env_test.cc
|
|
|
|
|
util/event_logger_test.cc
|
|
|
|
|
util/filelock_test.cc
|
RangeSync not to sync last 1MB of the file
Summary:
From other ones' investigation:
"sync_file_range() behavior highly depends on kernel version and filesystem.
xfs does neighbor page flushing outside of the specified ranges. For example, sync_file_range(fd, 8192, 16384) does not only trigger flushing page #3 to #4, but also flushing many more dirty pages (i.e. up to page#16)... Ranges of the sync_file_range() should be far enough from write() offset (at least 1MB)."
Test Plan: make all check
Reviewers: igor, rven, kradhakrishnan, yhchiang, IslamAbdelRahman, anthony
Reviewed By: anthony
Subscribers: yoshinorim, MarkCallaghan, sumeet, domas, dhruba, leveldb, ljin
Differential Revision: https://reviews.facebook.net/D15807
9 years ago
|
|
|
util/file_reader_writer_test.cc
|
|
|
|
|
util/heap_test.cc
|
|
|
|
|
util/histogram_test.cc
|
|
|
|
|
util/iostats_context_test.cc
|
|
|
|
|
util/mock_env_test.cc
|
|
|
|
|
util/options_settable_test.cc
|
|
|
|
|
util/options_test.cc
|
|
|
|
|
util/rate_limiter_test.cc
|
|
|
|
|
util/slice_transform_test.cc
|
|
|
|
|
util/statistics_test.cc
|
|
|
|
|
util/thread_list_test.cc
|
|
|
|
|
util/thread_local_test.cc
|
|
|
|
|
utilities/backupable/backupable_db_test.cc
|
|
|
|
|
utilities/checkpoint/checkpoint_test.cc
|
|
|
|
|
utilities/document/document_db_test.cc
|
|
|
|
|
utilities/document/json_document_test.cc
|
|
|
|
|
utilities/env_registry_test.cc
|
|
|
|
|
utilities/geodb/geodb_test.cc
|
|
|
|
|
utilities/memory/memory_test.cc
|
|
|
|
|
utilities/merge_operators/string_append/stringappend_test.cc
|
|
|
|
|
utilities/options/options_util_test.cc
|
|
|
|
|
utilities/persistent_cache/hash_table_test.cc
|
|
|
|
|
utilities/persistent_cache/persistent_cache_test.cc
|
|
|
|
|
utilities/redis/redis_lists_test.cc
|
|
|
|
|
utilities/spatialdb/spatial_db_test.cc
|
|
|
|
|
utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
|
|
|
|
|
utilities/transactions/optimistic_transaction_test.cc
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
10 years ago
|
|
|
utilities/transactions/transaction_test.cc
|
|
|
|
|
utilities/ttl/ttl_test.cc
|
|
|
|
|
utilities/write_batch_with_index/write_batch_with_index_test.cc
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
set(EXES ${APPS})
|
|
|
|
|
|
|
|
|
|
foreach(sourcefile ${EXES})
|
|
|
|
|
string(REPLACE ".cc" "" exename ${sourcefile})
|
|
|
|
|
string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
|
|
|
|
|
add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
|
|
|
|
|
target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${LIBS})
|
|
|
|
|
endforeach(sourcefile ${EXES})
|
|
|
|
|
|
|
|
|
|
# test utilities are only build in debug
|
|
|
|
|
set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
|
|
|
|
|
add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
|
|
|
|
|
set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
|
|
|
|
|
set_target_properties(${TESTUTILLIB}
|
|
|
|
|
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Tests are excluded from Release builds
|
|
|
|
|
set(TEST_EXES ${TESTS})
|
|
|
|
|
|
|
|
|
|
foreach(sourcefile ${TEST_EXES})
|
|
|
|
|
string(REPLACE ".cc" "" exename ${sourcefile})
|
|
|
|
|
string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
|
|
|
|
|
add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
|
|
|
|
|
set_target_properties(${exename}${ARTIFACT_SUFFIX}
|
|
|
|
|
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
|
|
|
|
|
)
|
|
|
|
|
target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${LIBS} testutillib${ARTIFACT_SUFFIX})
|
|
|
|
|
endforeach(sourcefile ${TEST_EXES})
|
|
|
|
|
|
|
|
|
|
# C executables must link to a shared object
|
|
|
|
|
set(C_TEST_EXES ${C_TESTS})
|
|
|
|
|
|
|
|
|
|
foreach(sourcefile ${C_TEST_EXES})
|
|
|
|
|
string(REPLACE ".c" "" exename ${sourcefile})
|
|
|
|
|
string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
|
|
|
|
|
add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
|
|
|
|
|
set_target_properties(${exename}${ARTIFACT_SUFFIX}
|
|
|
|
|
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
|
|
|
|
|
)
|
|
|
|
|
target_link_libraries(${exename}${ARTIFACT_SUFFIX} rocksdb${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX})
|
|
|
|
|
endforeach(sourcefile ${C_TEST_EXES})
|
