|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
//
|
|
|
|
// The representation of a DBImpl consists of a set of Versions. The
|
|
|
|
// newest version is called "current". Older versions may be kept
|
|
|
|
// around to provide a consistent view to live iterators.
|
|
|
|
//
|
|
|
|
// Each Version keeps track of a set of Table files per level. The
|
|
|
|
// entire set of versions is maintained in a VersionSet.
|
|
|
|
//
|
|
|
|
// Version,VersionSet are thread-compatible, but require external
|
|
|
|
// synchronization on all accesses.
|
|
|
|
|
|
|
|
#pragma once
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
10 years ago
|
|
|
#include <atomic>
|
|
|
|
#include <deque>
|
|
|
|
#include <limits>
|
|
|
|
#include <map>
|
|
|
|
#include <memory>
|
|
|
|
#include <set>
|
|
|
|
#include <string>
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
10 years ago
|
|
|
#include <utility>
|
|
|
|
#include <vector>
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
10 years ago
|
|
|
|
|
|
|
#include "db/column_family.h"
|
|
|
|
#include "db/compaction/compaction.h"
|
|
|
|
#include "db/compaction/compaction_picker.h"
|
|
|
|
#include "db/dbformat.h"
|
hints for narrowing down FindFile range and avoiding checking unrelevant L0 files
Summary:
The file tree structure in Version is prebuilt and the range of each file is known.
On the Get() code path, we do binary search in FindFile() by comparing
target key with each file's largest key and also check the range for each L0 file.
With some pre-calculated knowledge, each key comparision that has been done can serve
as a hint to narrow down further searches:
(1) If a key falls within a L0 file's range, we can safely skip the next
file if its range does not overlap with the current one.
(2) If a key falls within a file's range in level L0 - Ln-1, we should only
need to binary search in the next level for files that overlap with the current one.
(1) will be able to skip some files depending one the key distribution.
(2) can greatly reduce the range of binary search, especially for bottom
levels, given that one file most likely only overlaps with N files from
the level below (where N is max_bytes_for_level_multiplier). So on level
L, we will only look at ~N files instead of N^L files.
Some inital results: measured with 500M key DB, when write is light (10k/s = 1.2M/s), this
improves QPS ~7% on top of blocked bloom. When write is heavier (80k/s =
9.6M/s), it gives us ~13% improvement.
Test Plan: make all check
Reviewers: haobo, igor, dhruba, sdong, yhchiang
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D17205
11 years ago
|
|
|
#include "db/file_indexer.h"
|
|
|
|
#include "db/log_reader.h"
|
|
|
|
#include "db/range_del_aggregator.h"
|
|
|
|
#include "db/read_callback.h"
|
|
|
|
#include "db/table_cache.h"
|
|
|
|
#include "db/version_builder.h"
|
|
|
|
#include "db/version_edit.h"
|
Push- instead of pull-model for managing Write stalls
Summary:
Introducing WriteController, which is a source of truth about per-DB write delays. Let's define an DB epoch as a period where there are no flushes and compactions (i.e. new epoch is started when flush or compaction finishes). Each epoch can either:
* proceed with all writes without delay
* delay all writes by fixed time
* stop all writes
The three modes are recomputed at each epoch change (flush, compaction), rather than on every write (which is currently the case).
When we have a lot of column families, our current pull behavior adds a big overhead, since we need to loop over every column family for every write. With new push model, overhead on Write code-path is minimal.
This is just the start. Next step is to also take care of stalls introduced by slow memtable flushes. The final goal is to eliminate function MakeRoomForWrite(), which currently needs to be called for every column family by every write.
Test Plan: make check for now. I'll add some unit tests later. Also, perf test.
Reviewers: dhruba, yhchiang, MarkCallaghan, sdong, ljin
Reviewed By: ljin
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D22791
10 years ago
|
|
|
#include "db/write_controller.h"
|
|
|
|
#include "monitoring/instrumented_mutex.h"
|
|
|
|
#include "options/db_options.h"
|
|
|
|
#include "port/port.h"
|
|
|
|
#include "rocksdb/env.h"
|
Introduce a new MultiGet batching implementation (#5011)
Summary:
This PR introduces a new MultiGet() API, with the underlying implementation grouping keys based on SST file and batching lookups in a file. The reason for the new API is twofold - the definition allows callers to allocate storage for status and values on stack instead of std::vector, as well as return values as PinnableSlices in order to avoid copying, and it keeps the original MultiGet() implementation intact while we experiment with batching.
Batching is useful when there is some spatial locality to the keys being queries, as well as larger batch sizes. The main benefits are due to -
1. Fewer function calls, especially to BlockBasedTableReader::MultiGet() and FullFilterBlockReader::KeysMayMatch()
2. Bloom filter cachelines can be prefetched, hiding the cache miss latency
The next step is to optimize the binary searches in the level_storage_info, index blocks and data blocks, since we could reduce the number of key comparisons if the keys are relatively close to each other. The batching optimizations also need to be extended to other formats, such as PlainTable and filter formats. This also needs to be added to db_stress.
Benchmark results from db_bench for various batch size/locality of reference combinations are given below. Locality was simulated by offsetting the keys in a batch by a stride length. Each SST file is about 8.6MB uncompressed and key/value size is 16/100 uncompressed. To focus on the cpu benefit of batching, the runs were single threaded and bound to the same cpu to eliminate interference from other system events. The results show a 10-25% improvement in micros/op from smaller to larger batch sizes (4 - 32).
Batch Sizes
1 | 2 | 4 | 8 | 16 | 32
Random pattern (Stride length 0)
4.158 | 4.109 | 4.026 | 4.05 | 4.1 | 4.074 - Get
4.438 | 4.302 | 4.165 | 4.122 | 4.096 | 4.075 - MultiGet (no batching)
4.461 | 4.256 | 4.277 | 4.11 | 4.182 | 4.14 - MultiGet (w/ batching)
Good locality (Stride length 16)
4.048 | 3.659 | 3.248 | 2.99 | 2.84 | 2.753
4.429 | 3.728 | 3.406 | 3.053 | 2.911 | 2.781
4.452 | 3.45 | 2.833 | 2.451 | 2.233 | 2.135
Good locality (Stride length 256)
4.066 | 3.786 | 3.581 | 3.447 | 3.415 | 3.232
4.406 | 4.005 | 3.644 | 3.49 | 3.381 | 3.268
4.393 | 3.649 | 3.186 | 2.882 | 2.676 | 2.62
Medium locality (Stride length 4096)
4.012 | 3.922 | 3.768 | 3.61 | 3.582 | 3.555
4.364 | 4.057 | 3.791 | 3.65 | 3.57 | 3.465
4.479 | 3.758 | 3.316 | 3.077 | 2.959 | 2.891
dbbench command used (on a DB with 4 levels, 12 million keys)-
TEST_TMPDIR=/dev/shm numactl -C 10 ./db_bench.tmp -use_existing_db=true -benchmarks="readseq,multireadrandom" -write_buffer_size=4194304 -target_file_size_base=4194304 -max_bytes_for_level_base=16777216 -num=12000000 -reads=12000000 -duration=90 -threads=1 -compression_type=none -cache_size=4194304000 -batch_size=32 -disable_auto_compactions=true -bloom_bits=10 -cache_index_and_filter_blocks=true -pin_l0_filter_and_index_blocks_in_cache=true -multiread_batched=true -multiread_stride=4
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5011
Differential Revision: D14348703
Pulled By: anand1976
fbshipit-source-id: 774406dab3776d979c809522a67bedac6c17f84b
6 years ago
|
|
|
#include "table/get_context.h"
|
|
|
|
#include "table/multiget_context.h"
|
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
|
|
|
namespace log {
|
|
|
|
class Writer;
|
|
|
|
}
|
|
|
|
|
|
|
|
class Compaction;
|
|
|
|
class LogBuffer;
|
|
|
|
class LookupKey;
|
|
|
|
class MemTable;
|
|
|
|
class Version;
|
|
|
|
class VersionSet;
|
|
|
|
class WriteBufferManager;
|
|
|
|
class MergeContext;
|
|
|
|
class ColumnFamilySet;
|
|
|
|
class TableCache;
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
11 years ago
|
|
|
class MergeIteratorBuilder;
|
|
|
|
|
create compressed_levels_ in Version, allocate its space using arena. Make Version::Get, Version::FindFile faster
Summary:
Define CompressedFileMetaData that just contains fd, smallest_slice, largest_slice. Create compressed_levels_ in Version, the space is allocated using arena
Thus increase the file meta data locality, speed up "Get" and "FindFile"
benchmark with in-memory tmpfs, could have 4% improvement under "random read" and 2% improvement under "read while writing"
benchmark command:
./db_bench --db=/mnt/db/rocksdb --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --block_size=4096 --cache_size=17179869184 --cache_numshardbits=6 --compression_type=none --compression_ratio=1 --min_level_to_compress=-1 --disable_seek_compaction=1 --hard_rate_limit=2 --write_buffer_size=134217728 --max_write_buffer_number=2 --level0_file_num_compaction_trigger=8 --target_file_size_base=33554432 --max_bytes_for_level_base=1073741824 --disable_wal=0 --sync=0 --disable_data_sync=1 --verify_checksum=1 --delete_obsolete_files_period_micros=314572800 --max_grandparent_overlap_factor=10 --max_background_compactions=4 --max_background_flushes=0 --level0_slowdown_writes_trigger=16 --level0_stop_writes_trigger=24 --statistics=0 --stats_per_interval=0 --stats_interval=1048576 --histogram=0 --use_plain_table=1 --open_files=-1 --mmap_read=1 --mmap_write=0 --memtablerep=prefix_hash --bloom_bits=10 --bloom_locality=1 --perf_level=0 --benchmarks=readwhilewriting,readwhilewriting,readwhilewriting --use_existing_db=1 --num=52428800 --threads=1 —writes_per_second=81920
Read Random:
From 1.8363 ms/op, improve to 1.7587 ms/op.
Read while writing:
From 2.985 ms/op, improve to 2.924 ms/op.
Test Plan:
make all check
Reviewers: ljin, haobo, yhchiang, sdong
Reviewed By: sdong
Subscribers: dhruba, igor
Differential Revision: https://reviews.facebook.net/D19419
11 years ago
|
|
|
// Return the smallest index i such that file_level.files[i]->largest >= key.
|
|
|
|
// Return file_level.num_files if there is no such file.
|
|
|
|
// REQUIRES: "file_level.files" contains a sorted list of
|
|
|
|
// non-overlapping files.
|
|
|
|
extern int FindFile(const InternalKeyComparator& icmp,
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
10 years ago
|
|
|
const LevelFilesBrief& file_level, const Slice& key);
|
create compressed_levels_ in Version, allocate its space using arena. Make Version::Get, Version::FindFile faster
Summary:
Define CompressedFileMetaData that just contains fd, smallest_slice, largest_slice. Create compressed_levels_ in Version, the space is allocated using arena
Thus increase the file meta data locality, speed up "Get" and "FindFile"
benchmark with in-memory tmpfs, could have 4% improvement under "random read" and 2% improvement under "read while writing"
benchmark command:
./db_bench --db=/mnt/db/rocksdb --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --block_size=4096 --cache_size=17179869184 --cache_numshardbits=6 --compression_type=none --compression_ratio=1 --min_level_to_compress=-1 --disable_seek_compaction=1 --hard_rate_limit=2 --write_buffer_size=134217728 --max_write_buffer_number=2 --level0_file_num_compaction_trigger=8 --target_file_size_base=33554432 --max_bytes_for_level_base=1073741824 --disable_wal=0 --sync=0 --disable_data_sync=1 --verify_checksum=1 --delete_obsolete_files_period_micros=314572800 --max_grandparent_overlap_factor=10 --max_background_compactions=4 --max_background_flushes=0 --level0_slowdown_writes_trigger=16 --level0_stop_writes_trigger=24 --statistics=0 --stats_per_interval=0 --stats_interval=1048576 --histogram=0 --use_plain_table=1 --open_files=-1 --mmap_read=1 --mmap_write=0 --memtablerep=prefix_hash --bloom_bits=10 --bloom_locality=1 --perf_level=0 --benchmarks=readwhilewriting,readwhilewriting,readwhilewriting --use_existing_db=1 --num=52428800 --threads=1 —writes_per_second=81920
Read Random:
From 1.8363 ms/op, improve to 1.7587 ms/op.
Read while writing:
From 2.985 ms/op, improve to 2.924 ms/op.
Test Plan:
make all check
Reviewers: ljin, haobo, yhchiang, sdong
Reviewed By: sdong
Subscribers: dhruba, igor
Differential Revision: https://reviews.facebook.net/D19419
11 years ago
|
|
|
|
|
|
|
// Returns true iff some file in "files" overlaps the user key range
|
|
|
|
// [*smallest,*largest].
|
|
|
|
// smallest==nullptr represents a key smaller than all keys in the DB.
|
|
|
|
// largest==nullptr represents a key largest than all keys in the DB.
|
create compressed_levels_ in Version, allocate its space using arena. Make Version::Get, Version::FindFile faster
Summary:
Define CompressedFileMetaData that just contains fd, smallest_slice, largest_slice. Create compressed_levels_ in Version, the space is allocated using arena
Thus increase the file meta data locality, speed up "Get" and "FindFile"
benchmark with in-memory tmpfs, could have 4% improvement under "random read" and 2% improvement under "read while writing"
benchmark command:
./db_bench --db=/mnt/db/rocksdb --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --block_size=4096 --cache_size=17179869184 --cache_numshardbits=6 --compression_type=none --compression_ratio=1 --min_level_to_compress=-1 --disable_seek_compaction=1 --hard_rate_limit=2 --write_buffer_size=134217728 --max_write_buffer_number=2 --level0_file_num_compaction_trigger=8 --target_file_size_base=33554432 --max_bytes_for_level_base=1073741824 --disable_wal=0 --sync=0 --disable_data_sync=1 --verify_checksum=1 --delete_obsolete_files_period_micros=314572800 --max_grandparent_overlap_factor=10 --max_background_compactions=4 --max_background_flushes=0 --level0_slowdown_writes_trigger=16 --level0_stop_writes_trigger=24 --statistics=0 --stats_per_interval=0 --stats_interval=1048576 --histogram=0 --use_plain_table=1 --open_files=-1 --mmap_read=1 --mmap_write=0 --memtablerep=prefix_hash --bloom_bits=10 --bloom_locality=1 --perf_level=0 --benchmarks=readwhilewriting,readwhilewriting,readwhilewriting --use_existing_db=1 --num=52428800 --threads=1 —writes_per_second=81920
Read Random:
From 1.8363 ms/op, improve to 1.7587 ms/op.
Read while writing:
From 2.985 ms/op, improve to 2.924 ms/op.
Test Plan:
make all check
Reviewers: ljin, haobo, yhchiang, sdong
Reviewed By: sdong
Subscribers: dhruba, igor
Differential Revision: https://reviews.facebook.net/D19419
11 years ago
|
|
|
// REQUIRES: If disjoint_sorted_files, file_level.files[]
|
|
|
|
// contains disjoint ranges in sorted order.
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
10 years ago
|
|
|
extern bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
|
|
|
|
bool disjoint_sorted_files,
|
|
|
|
const LevelFilesBrief& file_level,
|
|
|
|
const Slice* smallest_user_key,
|
|
|
|
const Slice* largest_user_key);
|
|
|
|
|
|
|
|
// Generate LevelFilesBrief from vector<FdWithKeyRange*>
|
|
|
|
// Would copy smallest_key and largest_key data to sequential memory
|
|
|
|
// arena: Arena used to allocate the memory
|
|
|
|
extern void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
10 years ago
|
|
|
const std::vector<FileMetaData*>& files,
|
|
|
|
Arena* arena);
|
|
|
|
|
|
|
|
// Information of the storage associated with each Version, including number of
|
|
|
|
// levels of LSM tree, files information at each level, files marked for
|
|
|
|
// compaction, etc.
|
|
|
|
class VersionStorageInfo {
|
|
|
|
public:
|
|
|
|
VersionStorageInfo(const InternalKeyComparator* internal_comparator,
|
|
|
|
const Comparator* user_comparator, int num_levels,
|
|
|
|
CompactionStyle compaction_style,
|
|
|
|
VersionStorageInfo* src_vstorage,
|
|
|
|
bool _force_consistency_checks);
|
|
|
|
~VersionStorageInfo();
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
11 years ago
|
|
|
|
|
|
|
void Reserve(int level, size_t size) { files_[level].reserve(size); }
|
|
|
|
|
|
|
|
void AddFile(int level, FileMetaData* f, Logger* info_log = nullptr);
|
|
|
|
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
10 years ago
|
|
|
void SetFinalized();
|
|
|
|
|
|
|
|
// Update num_non_empty_levels_.
|
|
|
|
void UpdateNumNonEmptyLevels();
|
|
|
|
|
|
|
|
void GenerateFileIndexer() {
|
|
|
|
file_indexer_.UpdateIndex(&arena_, num_non_empty_levels_, files_);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Update the accumulated stats from a file-meta.
|
|
|
|
void UpdateAccumulatedStats(FileMetaData* file_meta);
|
|
|
|
|
|
|
|
// Decrease the current stat from a to-be-deleted file-meta
|
|
|
|
void RemoveCurrentStats(FileMetaData* file_meta);
|
|
|
|
|
|
|
|
void ComputeCompensatedSizes();
|
|
|
|
|
|
|
|
// Updates internal structures that keep track of compaction scores
|
|
|
|
// We use compaction scores to figure out which compaction to do next
|
|
|
|
// REQUIRES: db_mutex held!!
|
|
|
|
// TODO find a better way to pass compaction_options_fifo.
|
|
|
|
void ComputeCompactionScore(const ImmutableCFOptions& immutable_cf_options,
|
|
|
|
const MutableCFOptions& mutable_cf_options);
|
|
|
|
|
|
|
|
// Estimate est_comp_needed_bytes_
|
|
|
|
void EstimateCompactionBytesNeeded(
|
|
|
|
const MutableCFOptions& mutable_cf_options);
|
|
|
|
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
10 years ago
|
|
|
// This computes files_marked_for_compaction_ and is called by
|
|
|
|
// ComputeCompactionScore()
|
|
|
|
void ComputeFilesMarkedForCompaction();
|
|
|
|
|
|
|
|
// This computes ttl_expired_files_ and is called by
|
|
|
|
// ComputeCompactionScore()
|
|
|
|
void ComputeExpiredTtlFiles(const ImmutableCFOptions& ioptions,
|
|
|
|
const uint64_t ttl);
|
|
|
|
|
Periodic Compactions (#5166)
Summary:
Introducing Periodic Compactions.
This feature allows all the files in a CF to be periodically compacted. It could help in catching any corruptions that could creep into the DB proactively as every file is constantly getting re-compacted. And also, of course, it helps to cleanup data older than certain threshold.
- Introduced a new option `periodic_compaction_time` to control how long a file can live without being compacted in a CF.
- This works across all levels.
- The files are put in the same level after going through the compaction. (Related files in the same level are picked up as `ExpandInputstoCleanCut` is used).
- Compaction filters, if any, are invoked as usual.
- A new table property, `file_creation_time`, is introduced to implement this feature. This property is set to the time at which the SST file was created (and that time is given by the underlying Env/OS).
This feature can be enabled on its own, or in conjunction with `ttl`. It is possible to set a different time threshold for the bottom level when used in conjunction with ttl. Since `ttl` works only on 0 to last but one levels, you could set `ttl` to, say, 1 day, and `periodic_compaction_time` to, say, 7 days. Since `ttl < periodic_compaction_time` all files in last but one levels keep getting picked up based on ttl, and almost never based on periodic_compaction_time. The files in the bottom level get picked up for compaction based on `periodic_compaction_time`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5166
Differential Revision: D14884441
Pulled By: sagar0
fbshipit-source-id: 408426cbacb409c06386a98632dcf90bfa1bda47
6 years ago
|
|
|
// This computes files_marked_for_periodic_compaction_ and is called by
|
|
|
|
// ComputeCompactionScore()
|
|
|
|
void ComputeFilesMarkedForPeriodicCompaction(
|
|
|
|
const ImmutableCFOptions& ioptions,
|
|
|
|
const uint64_t periodic_compaction_seconds);
|
|
|
|
|
|
|
|
// This computes bottommost_files_marked_for_compaction_ and is called by
|
|
|
|
// ComputeCompactionScore() or UpdateOldestSnapshot().
|
|
|
|
//
|
|
|
|
// Among bottommost files (assumes they've already been computed), marks the
|
|
|
|
// ones that have keys that would be eliminated if recompacted, according to
|
|
|
|
// the seqnum of the oldest existing snapshot. Must be called every time
|
|
|
|
// oldest snapshot changes as that is when bottom-level files can become
|
|
|
|
// eligible for compaction.
|
|
|
|
//
|
|
|
|
// REQUIRES: DB mutex held
|
|
|
|
void ComputeBottommostFilesMarkedForCompaction();
|
|
|
|
|
|
|
|
// Generate level_files_brief_ from files_
|
|
|
|
void GenerateLevelFilesBrief();
|
|
|
|
// Sort all files for this version based on their file size and
|
|
|
|
// record results in files_by_compaction_pri_. The largest files are listed
|
|
|
|
// first.
|
|
|
|
void UpdateFilesByCompactionPri(CompactionPri compaction_pri);
|
create compressed_levels_ in Version, allocate its space using arena. Make Version::Get, Version::FindFile faster
Summary:
Define CompressedFileMetaData that just contains fd, smallest_slice, largest_slice. Create compressed_levels_ in Version, the space is allocated using arena
Thus increase the file meta data locality, speed up "Get" and "FindFile"
benchmark with in-memory tmpfs, could have 4% improvement under "random read" and 2% improvement under "read while writing"
benchmark command:
./db_bench --db=/mnt/db/rocksdb --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --block_size=4096 --cache_size=17179869184 --cache_numshardbits=6 --compression_type=none --compression_ratio=1 --min_level_to_compress=-1 --disable_seek_compaction=1 --hard_rate_limit=2 --write_buffer_size=134217728 --max_write_buffer_number=2 --level0_file_num_compaction_trigger=8 --target_file_size_base=33554432 --max_bytes_for_level_base=1073741824 --disable_wal=0 --sync=0 --disable_data_sync=1 --verify_checksum=1 --delete_obsolete_files_period_micros=314572800 --max_grandparent_overlap_factor=10 --max_background_compactions=4 --max_background_flushes=0 --level0_slowdown_writes_trigger=16 --level0_stop_writes_trigger=24 --statistics=0 --stats_per_interval=0 --stats_interval=1048576 --histogram=0 --use_plain_table=1 --open_files=-1 --mmap_read=1 --mmap_write=0 --memtablerep=prefix_hash --bloom_bits=10 --bloom_locality=1 --perf_level=0 --benchmarks=readwhilewriting,readwhilewriting,readwhilewriting --use_existing_db=1 --num=52428800 --threads=1 —writes_per_second=81920
Read Random:
From 1.8363 ms/op, improve to 1.7587 ms/op.
Read while writing:
From 2.985 ms/op, improve to 2.924 ms/op.
Test Plan:
make all check
Reviewers: ljin, haobo, yhchiang, sdong
Reviewed By: sdong
Subscribers: dhruba, igor
Differential Revision: https://reviews.facebook.net/D19419
11 years ago
|
|
|
|
Allowing L0 -> L1 trivial move on sorted data
Summary:
This diff updates the logic of how we do trivial move, now trivial move can run on any number of files in input level as long as they are not overlapping
The conditions for trivial move have been updated
Introduced conditions:
- Trivial move cannot happen if we have a compaction filter (except if the compaction is not manual)
- Input level files cannot be overlapping
Removed conditions:
- Trivial move only run when the compaction is not manual
- Input level should can contain only 1 file
More context on what tests failed because of Trivial move
```
DBTest.CompactionsGenerateMultipleFiles
This test is expecting compaction on a file in L0 to generate multiple files in L1, this test will fail with trivial move because we end up with one file in L1
```
```
DBTest.NoSpaceCompactRange
This test expect compaction to fail when we force environment to report running out of space, of course this is not valid in trivial move situation
because trivial move does not need any extra space, and did not check for that
```
```
DBTest.DropWrites
Similar to DBTest.NoSpaceCompactRange
```
```
DBTest.DeleteObsoleteFilesPendingOutputs
This test expect that a file in L2 is deleted after it's moved to L3, this is not valid with trivial move because although the file was moved it is now used by L3
```
```
CuckooTableDBTest.CompactionIntoMultipleFiles
Same as DBTest.CompactionsGenerateMultipleFiles
```
This diff is based on a work by @sdong https://reviews.facebook.net/D34149
Test Plan: make -j64 check
Reviewers: rven, sdong, igor
Reviewed By: igor
Subscribers: yhchiang, ott, march, dhruba, sdong
Differential Revision: https://reviews.facebook.net/D34797
10 years ago
|
|
|
void GenerateLevel0NonOverlapping();
|
|
|
|
bool level0_non_overlapping() const {
|
|
|
|
return level0_non_overlapping_;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check whether each file in this version is bottommost (i.e., nothing in its
|
|
|
|
// key-range could possibly exist in an older file/level).
|
|
|
|
// REQUIRES: This version has not been saved
|
|
|
|
void GenerateBottommostFiles();
|
|
|
|
|
|
|
|
// Updates the oldest snapshot and related internal state, like the bottommost
|
|
|
|
// files marked for compaction.
|
|
|
|
// REQUIRES: DB mutex held
|
|
|
|
void UpdateOldestSnapshot(SequenceNumber oldest_snapshot_seqnum);
|
|
|
|
|
|
|
|
int MaxInputLevel() const;
|
|
|
|
int MaxOutputLevel(bool allow_ingest_behind) const;
|
|
|
|
|
|
|
|
// Return level number that has idx'th highest score
|
|
|
|
int CompactionScoreLevel(int idx) const { return compaction_level_[idx]; }
|
|
|
|
|
|
|
|
// Return idx'th highest score
|
|
|
|
double CompactionScore(int idx) const { return compaction_score_[idx]; }
|
|
|
|
|
|
|
|
void GetOverlappingInputs(
|
|
|
|
int level, const InternalKey* begin, // nullptr means before all keys
|
|
|
|
const InternalKey* end, // nullptr means after all keys
|
|
|
|
std::vector<FileMetaData*>* inputs,
|
|
|
|
int hint_index = -1, // index of overlap file
|
|
|
|
int* file_index = nullptr, // return index of overlap file
|
|
|
|
bool expand_range = true, // if set, returns files which overlap the
|
|
|
|
// range and overlap each other. If false,
|
|
|
|
// then just files intersecting the range
|
|
|
|
InternalKey** next_smallest = nullptr) // if non-null, returns the
|
|
|
|
const; // smallest key of next file not included
|
level compaction expansion
Summary:
reimplement the compaction expansion on lower level.
Considering such a case:
input level file: 1[B E] 2[F G] 3[H I] 4 [J M]
output level file: 5[A C] 6[D K] 7[L O]
If we initially pick file 2, now we will compact file 2 and 6. But we can safely compact 2, 3 and 6 without expanding the output level.
The previous code is messy and wrong.
In this diff, I first determine the input range [a, b], and output range [c, d],
then we get the range [e,f] = [min(a, c), max(b, d] and put all eligible clean-cut files within [e, f] into this compaction.
**Note: clean-cut means the files don't have the same user key on the boundaries of some files that are not chosen in this compaction**.
Closes https://github.com/facebook/rocksdb/pull/1760
Differential Revision: D4395564
Pulled By: lightmark
fbshipit-source-id: 2dc2c5c
8 years ago
|
|
|
void GetCleanInputsWithinInterval(
|
|
|
|
int level, const InternalKey* begin, // nullptr means before all keys
|
|
|
|
const InternalKey* end, // nullptr means after all keys
|
|
|
|
std::vector<FileMetaData*>* inputs,
|
|
|
|
int hint_index = -1, // index of overlap file
|
|
|
|
int* file_index = nullptr) // return index of overlap file
|
|
|
|
const;
|
|
|
|
|
level compaction expansion
Summary:
reimplement the compaction expansion on lower level.
Considering such a case:
input level file: 1[B E] 2[F G] 3[H I] 4 [J M]
output level file: 5[A C] 6[D K] 7[L O]
If we initially pick file 2, now we will compact file 2 and 6. But we can safely compact 2, 3 and 6 without expanding the output level.
The previous code is messy and wrong.
In this diff, I first determine the input range [a, b], and output range [c, d],
then we get the range [e,f] = [min(a, c), max(b, d] and put all eligible clean-cut files within [e, f] into this compaction.
**Note: clean-cut means the files don't have the same user key on the boundaries of some files that are not chosen in this compaction**.
Closes https://github.com/facebook/rocksdb/pull/1760
Differential Revision: D4395564
Pulled By: lightmark
fbshipit-source-id: 2dc2c5c
8 years ago
|
|
|
void GetOverlappingInputsRangeBinarySearch(
|
|
|
|
int level, // level > 0
|
|
|
|
const InternalKey* begin, // nullptr means before all keys
|
|
|
|
const InternalKey* end, // nullptr means after all keys
|
|
|
|
std::vector<FileMetaData*>* inputs,
|
level compaction expansion
Summary:
reimplement the compaction expansion on lower level.
Considering such a case:
input level file: 1[B E] 2[F G] 3[H I] 4 [J M]
output level file: 5[A C] 6[D K] 7[L O]
If we initially pick file 2, now we will compact file 2 and 6. But we can safely compact 2, 3 and 6 without expanding the output level.
The previous code is messy and wrong.
In this diff, I first determine the input range [a, b], and output range [c, d],
then we get the range [e,f] = [min(a, c), max(b, d] and put all eligible clean-cut files within [e, f] into this compaction.
**Note: clean-cut means the files don't have the same user key on the boundaries of some files that are not chosen in this compaction**.
Closes https://github.com/facebook/rocksdb/pull/1760
Differential Revision: D4395564
Pulled By: lightmark
fbshipit-source-id: 2dc2c5c
8 years ago
|
|
|
int hint_index, // index of overlap file
|
|
|
|
int* file_index, // return index of overlap file
|
|
|
|
bool within_interval = false, // if set, force the inputs within interval
|
|
|
|
InternalKey** next_smallest = nullptr) // if non-null, returns the
|
|
|
|
const; // smallest key of next file not included
|
|
|
|
|
|
|
|
// Returns true iff some file in the specified level overlaps
|
|
|
|
// some part of [*smallest_user_key,*largest_user_key].
|
|
|
|
// smallest_user_key==NULL represents a key smaller than all keys in the DB.
|
|
|
|
// largest_user_key==NULL represents a key largest than all keys in the DB.
|
|
|
|
bool OverlapInLevel(int level, const Slice* smallest_user_key,
|
|
|
|
const Slice* largest_user_key);
|
|
|
|
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
12 years ago
|
|
|
// Returns true iff the first or last file in inputs contains
|
|
|
|
// an overlapping user key to the file "just outside" of it (i.e.
|
|
|
|
// just after the last file, or just before the first file)
|
|
|
|
// REQUIRES: "*inputs" is a sorted list of non-overlapping files
|
|
|
|
bool HasOverlappingUserKey(const std::vector<FileMetaData*>* inputs,
|
|
|
|
int level);
|
|
|
|
|
|
|
|
int num_levels() const { return num_levels_; }
|
|
|
|
|
|
|
|
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
|
|
|
int num_non_empty_levels() const {
|
|
|
|
assert(finalized_);
|
|
|
|
return num_non_empty_levels_;
|
|
|
|
}
|
|
|
|
|
|
|
|
// REQUIRES: This version has been finalized.
|
|
|
|
// (CalculateBaseBytes() is called)
|
|
|
|
// This may or may not return number of level files. It is to keep backward
|
|
|
|
// compatible behavior in universal compaction.
|
|
|
|
int l0_delay_trigger_count() const { return l0_delay_trigger_count_; }
|
|
|
|
|
|
|
|
void set_l0_delay_trigger_count(int v) { l0_delay_trigger_count_ = v; }
|
|
|
|
|
|
|
|
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
|
|
|
int NumLevelFiles(int level) const {
|
|
|
|
assert(finalized_);
|
|
|
|
return static_cast<int>(files_[level].size());
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return the combined file size of all files at the specified level.
|
|
|
|
uint64_t NumLevelBytes(int level) const;
|
|
|
|
|
|
|
|
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
|
|
|
const std::vector<FileMetaData*>& LevelFiles(int level) const {
|
|
|
|
return files_[level];
|
|
|
|
}
|
|
|
|
|
|
|
|
const rocksdb::LevelFilesBrief& LevelFilesBrief(int level) const {
|
|
|
|
assert(level < static_cast<int>(level_files_brief_.size()));
|
|
|
|
return level_files_brief_[level];
|
|
|
|
}
|
|
|
|
|
|
|
|
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
|
|
|
const std::vector<int>& FilesByCompactionPri(int level) const {
|
|
|
|
assert(finalized_);
|
|
|
|
return files_by_compaction_pri_[level];
|
|
|
|
}
|
|
|
|
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
10 years ago
|
|
|
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
|
|
|
// REQUIRES: DB mutex held during access
|
|
|
|
const autovector<std::pair<int, FileMetaData*>>& FilesMarkedForCompaction()
|
|
|
|
const {
|
|
|
|
assert(finalized_);
|
|
|
|
return files_marked_for_compaction_;
|
|
|
|
}
|
|
|
|
|
|
|
|
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
|
|
|
// REQUIRES: DB mutex held during access
|
|
|
|
const autovector<std::pair<int, FileMetaData*>>& ExpiredTtlFiles() const {
|
|
|
|
assert(finalized_);
|
|
|
|
return expired_ttl_files_;
|
|
|
|
}
|
|
|
|
|
Periodic Compactions (#5166)
Summary:
Introducing Periodic Compactions.
This feature allows all the files in a CF to be periodically compacted. It could help in catching any corruptions that could creep into the DB proactively as every file is constantly getting re-compacted. And also, of course, it helps to cleanup data older than certain threshold.
- Introduced a new option `periodic_compaction_time` to control how long a file can live without being compacted in a CF.
- This works across all levels.
- The files are put in the same level after going through the compaction. (Related files in the same level are picked up as `ExpandInputstoCleanCut` is used).
- Compaction filters, if any, are invoked as usual.
- A new table property, `file_creation_time`, is introduced to implement this feature. This property is set to the time at which the SST file was created (and that time is given by the underlying Env/OS).
This feature can be enabled on its own, or in conjunction with `ttl`. It is possible to set a different time threshold for the bottom level when used in conjunction with ttl. Since `ttl` works only on 0 to last but one levels, you could set `ttl` to, say, 1 day, and `periodic_compaction_time` to, say, 7 days. Since `ttl < periodic_compaction_time` all files in last but one levels keep getting picked up based on ttl, and almost never based on periodic_compaction_time. The files in the bottom level get picked up for compaction based on `periodic_compaction_time`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5166
Differential Revision: D14884441
Pulled By: sagar0
fbshipit-source-id: 408426cbacb409c06386a98632dcf90bfa1bda47
6 years ago
|
|
|
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
|
|
|
// REQUIRES: DB mutex held during access
|
|
|
|
const autovector<std::pair<int, FileMetaData*>>&
|
|
|
|
FilesMarkedForPeriodicCompaction() const {
|
|
|
|
assert(finalized_);
|
|
|
|
return files_marked_for_periodic_compaction_;
|
|
|
|
}
|
|
|
|
|
|
|
|
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
|
|
|
// REQUIRES: DB mutex held during access
|
|
|
|
const autovector<std::pair<int, FileMetaData*>>&
|
|
|
|
BottommostFilesMarkedForCompaction() const {
|
|
|
|
assert(finalized_);
|
|
|
|
return bottommost_files_marked_for_compaction_;
|
|
|
|
}
|
|
|
|
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
10 years ago
|
|
|
int base_level() const { return base_level_; }
|
|
|
|
double level_multiplier() const { return level_multiplier_; }
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
10 years ago
|
|
|
|
|
|
|
// REQUIRES: lock is held
|
|
|
|
// Set the index that is used to offset into files_by_compaction_pri_ to find
|
|
|
|
// the next compaction candidate file.
|
|
|
|
void SetNextCompactionIndex(int level, int index) {
|
|
|
|
next_file_to_compact_by_size_[level] = index;
|
|
|
|
}
|
|
|
|
|
|
|
|
// REQUIRES: lock is held
|
|
|
|
int NextCompactionIndex(int level) const {
|
|
|
|
return next_file_to_compact_by_size_[level];
|
|
|
|
}
|
|
|
|
|
|
|
|
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
|
|
|
const FileIndexer& file_indexer() const {
|
|
|
|
assert(finalized_);
|
|
|
|
return file_indexer_;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Only the first few entries of files_by_compaction_pri_ are sorted.
|
|
|
|
// There is no need to sort all the files because it is likely
|
|
|
|
// that on a running system, we need to look at only the first
|
|
|
|
// few largest files because a new version is created every few
|
|
|
|
// seconds/minutes (because of concurrent compactions).
|
|
|
|
static const size_t kNumberFilesToSort = 50;
|
|
|
|
|
|
|
|
// Return a human-readable short (single-line) summary of the number
|
|
|
|
// of files per level. Uses *scratch as backing store.
|
|
|
|
struct LevelSummaryStorage {
|
|
|
|
char buffer[1000];
|
|
|
|
};
|
|
|
|
struct FileSummaryStorage {
|
|
|
|
char buffer[3000];
|
|
|
|
};
|
|
|
|
const char* LevelSummary(LevelSummaryStorage* scratch) const;
|
|
|
|
// Return a human-readable short (single-line) summary of files
|
|
|
|
// in a specified level. Uses *scratch as backing store.
|
|
|
|
const char* LevelFileSummary(FileSummaryStorage* scratch, int level) const;
|
|
|
|
|
|
|
|
// Return the maximum overlapping data (in bytes) at next level for any
|
|
|
|
// file at a level >= 1.
|
|
|
|
int64_t MaxNextLevelOverlappingBytes();
|
|
|
|
|
|
|
|
// Return a human readable string that describes this version's contents.
|
|
|
|
std::string DebugString(bool hex = false) const;
|
|
|
|
|
|
|
|
uint64_t GetAverageValueSize() const {
|
|
|
|
if (accumulated_num_non_deletions_ == 0) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
assert(accumulated_raw_key_size_ + accumulated_raw_value_size_ > 0);
|
|
|
|
assert(accumulated_file_size_ > 0);
|
|
|
|
return accumulated_raw_value_size_ / accumulated_num_non_deletions_ *
|
|
|
|
accumulated_file_size_ /
|
|
|
|
(accumulated_raw_key_size_ + accumulated_raw_value_size_);
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t GetEstimatedActiveKeys() const;
|
|
|
|
|
|
|
|
double GetEstimatedCompressionRatioAtLevel(int level) const;
|
|
|
|
|
|
|
|
// re-initializes the index that is used to offset into
|
|
|
|
// files_by_compaction_pri_
|
|
|
|
// to find the next compaction candidate file.
|
|
|
|
void ResetNextCompactionIndex(int level) {
|
|
|
|
next_file_to_compact_by_size_[level] = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
const InternalKeyComparator* InternalComparator() {
|
|
|
|
return internal_comparator_;
|
|
|
|
}
|
|
|
|
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
10 years ago
|
|
|
// Returns maximum total bytes of data on a given level.
|
|
|
|
uint64_t MaxBytesForLevel(int level) const;
|
|
|
|
|
|
|
|
// Must be called after any change to MutableCFOptions.
|
|
|
|
void CalculateBaseBytes(const ImmutableCFOptions& ioptions,
|
|
|
|
const MutableCFOptions& options);
|
|
|
|
|
|
|
|
// Returns an estimate of the amount of live data in bytes.
|
|
|
|
uint64_t EstimateLiveDataSize() const;
|
|
|
|
|
|
|
|
uint64_t estimated_compaction_needed_bytes() const {
|
|
|
|
return estimated_compaction_needed_bytes_;
|
|
|
|
}
|
|
|
|
|
When slowdown is triggered, reduce the write rate
Summary: It's usually hard for users to set a value of options.delayed_write_rate. With this diff, after slowdown condition triggers, we greedily reduce write rate if estimated pending compaction bytes increase. If estimated compaction pending bytes drop, we increase the write rate.
Test Plan:
Add a unit test
Test with db_bench setting:
TEST_TMPDIR=/dev/shm/ ./db_bench --benchmarks=fillrandom -num=10000000 --soft_pending_compaction_bytes_limit=1000000000 --hard_pending_compaction_bytes_limit=3000000000 --delayed_write_rate=100000000
and make sure without the commit, write stop will happen, but with the commit, it will not happen.
Reviewers: igor, anthony, rven, yhchiang, kradhakrishnan, IslamAbdelRahman
Reviewed By: IslamAbdelRahman
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D52131
9 years ago
|
|
|
void TEST_set_estimated_compaction_needed_bytes(uint64_t v) {
|
|
|
|
estimated_compaction_needed_bytes_ = v;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool force_consistency_checks() const { return force_consistency_checks_; }
|
|
|
|
|
|
|
|
SequenceNumber bottommost_files_mark_threshold() const {
|
|
|
|
return bottommost_files_mark_threshold_;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Returns whether any key in [`smallest_key`, `largest_key`] could appear in
|
|
|
|
// an older L0 file than `last_l0_idx` or in a greater level than `last_level`
|
|
|
|
//
|
|
|
|
// @param last_level Level after which we check for overlap
|
|
|
|
// @param last_l0_idx If `last_level == 0`, index of L0 file after which we
|
|
|
|
// check for overlap; otherwise, must be -1
|
|
|
|
bool RangeMightExistAfterSortedRun(const Slice& smallest_user_key,
|
|
|
|
const Slice& largest_user_key,
|
|
|
|
int last_level, int last_l0_idx);
|
|
|
|
|
|
|
|
private:
|
|
|
|
const InternalKeyComparator* internal_comparator_;
|
|
|
|
const Comparator* user_comparator_;
|
|
|
|
int num_levels_; // Number of levels
|
|
|
|
int num_non_empty_levels_; // Number of levels. Any level larger than it
|
|
|
|
// is guaranteed to be empty.
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
10 years ago
|
|
|
// Per-level max bytes
|
|
|
|
std::vector<uint64_t> level_max_bytes_;
|
|
|
|
|
|
|
|
// A short brief metadata of files per level
|
|
|
|
autovector<rocksdb::LevelFilesBrief> level_files_brief_;
|
|
|
|
FileIndexer file_indexer_;
|
|
|
|
Arena arena_; // Used to allocate space for file_levels_
|
|
|
|
|
|
|
|
CompactionStyle compaction_style_;
|
|
|
|
|
|
|
|
// List of files per level, files in each level are arranged
|
|
|
|
// in increasing order of keys
|
|
|
|
std::vector<FileMetaData*>* files_;
|
|
|
|
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
10 years ago
|
|
|
// Level that L0 data should be compacted to. All levels < base_level_ should
|
|
|
|
// be empty. -1 if it is not level-compaction so it's not applicable.
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
10 years ago
|
|
|
int base_level_;
|
|
|
|
|
|
|
|
double level_multiplier_;
|
|
|
|
|
|
|
|
// A list for the same set of files that are stored in files_,
|
|
|
|
// but files in each level are now sorted based on file
|
|
|
|
// size. The file with the largest size is at the front.
|
|
|
|
// This vector stores the index of the file from files_.
|
|
|
|
std::vector<std::vector<int>> files_by_compaction_pri_;
|
|
|
|
|
Allowing L0 -> L1 trivial move on sorted data
Summary:
This diff updates the logic of how we do trivial move, now trivial move can run on any number of files in input level as long as they are not overlapping
The conditions for trivial move have been updated
Introduced conditions:
- Trivial move cannot happen if we have a compaction filter (except if the compaction is not manual)
- Input level files cannot be overlapping
Removed conditions:
- Trivial move only run when the compaction is not manual
- Input level should can contain only 1 file
More context on what tests failed because of Trivial move
```
DBTest.CompactionsGenerateMultipleFiles
This test is expecting compaction on a file in L0 to generate multiple files in L1, this test will fail with trivial move because we end up with one file in L1
```
```
DBTest.NoSpaceCompactRange
This test expect compaction to fail when we force environment to report running out of space, of course this is not valid in trivial move situation
because trivial move does not need any extra space, and did not check for that
```
```
DBTest.DropWrites
Similar to DBTest.NoSpaceCompactRange
```
```
DBTest.DeleteObsoleteFilesPendingOutputs
This test expect that a file in L2 is deleted after it's moved to L3, this is not valid with trivial move because although the file was moved it is now used by L3
```
```
CuckooTableDBTest.CompactionIntoMultipleFiles
Same as DBTest.CompactionsGenerateMultipleFiles
```
This diff is based on a work by @sdong https://reviews.facebook.net/D34149
Test Plan: make -j64 check
Reviewers: rven, sdong, igor
Reviewed By: igor
Subscribers: yhchiang, ott, march, dhruba, sdong
Differential Revision: https://reviews.facebook.net/D34797
10 years ago
|
|
|
// If true, means that files in L0 have keys with non overlapping ranges
|
|
|
|
bool level0_non_overlapping_;
|
|
|
|
|
|
|
|
// An index into files_by_compaction_pri_ that specifies the first
|
|
|
|
// file that is not yet compacted
|
|
|
|
std::vector<int> next_file_to_compact_by_size_;
|
|
|
|
|
|
|
|
// Only the first few entries of files_by_compaction_pri_ are sorted.
|
|
|
|
// There is no need to sort all the files because it is likely
|
|
|
|
// that on a running system, we need to look at only the first
|
|
|
|
// few largest files because a new version is created every few
|
|
|
|
// seconds/minutes (because of concurrent compactions).
|
|
|
|
static const size_t number_of_files_to_sort_ = 50;
|
|
|
|
|
Add experimental API MarkForCompaction()
Summary:
Some Mongo+Rocks datasets in Parse's environment are not doing compactions very frequently. During the quiet period (with no IO), we'd like to schedule compactions so that our reads become faster. Also, aggressively compacting during quiet periods helps when write bursts happen. In addition, we also want to compact files that are containing deleted key ranges (like old oplog keys).
All of this is currently not possible with CompactRange() because it's single-threaded and blocks all other compactions from happening. Running CompactRange() risks an issue of blocking writes because we generate too much Level 0 files before the compaction is over. Stopping writes is very dangerous because they hold transaction locks. We tried running manual compaction once on Mongo+Rocks and everything fell apart.
MarkForCompaction() solves all of those problems. This is very light-weight manual compaction. It is lower priority than automatic compactions, which means it shouldn't interfere with background process keeping the LSM tree clean. However, if no automatic compactions need to be run (or we have extra background threads available), we will start compacting files that are marked for compaction.
Test Plan: added a new unit test
Reviewers: yhchiang, rven, MarkCallaghan, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D37083
10 years ago
|
|
|
// This vector contains list of files marked for compaction and also not
|
|
|
|
// currently being compacted. It is protected by DB mutex. It is calculated in
|
|
|
|
// ComputeCompactionScore()
|
|
|
|
autovector<std::pair<int, FileMetaData*>> files_marked_for_compaction_;
|
|
|
|
|
|
|
|
autovector<std::pair<int, FileMetaData*>> expired_ttl_files_;
|
|
|
|
|
Periodic Compactions (#5166)
Summary:
Introducing Periodic Compactions.
This feature allows all the files in a CF to be periodically compacted. It could help in catching any corruptions that could creep into the DB proactively as every file is constantly getting re-compacted. And also, of course, it helps to cleanup data older than certain threshold.
- Introduced a new option `periodic_compaction_time` to control how long a file can live without being compacted in a CF.
- This works across all levels.
- The files are put in the same level after going through the compaction. (Related files in the same level are picked up as `ExpandInputstoCleanCut` is used).
- Compaction filters, if any, are invoked as usual.
- A new table property, `file_creation_time`, is introduced to implement this feature. This property is set to the time at which the SST file was created (and that time is given by the underlying Env/OS).
This feature can be enabled on its own, or in conjunction with `ttl`. It is possible to set a different time threshold for the bottom level when used in conjunction with ttl. Since `ttl` works only on 0 to last but one levels, you could set `ttl` to, say, 1 day, and `periodic_compaction_time` to, say, 7 days. Since `ttl < periodic_compaction_time` all files in last but one levels keep getting picked up based on ttl, and almost never based on periodic_compaction_time. The files in the bottom level get picked up for compaction based on `periodic_compaction_time`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5166
Differential Revision: D14884441
Pulled By: sagar0
fbshipit-source-id: 408426cbacb409c06386a98632dcf90bfa1bda47
6 years ago
|
|
|
autovector<std::pair<int, FileMetaData*>>
|
|
|
|
files_marked_for_periodic_compaction_;
|
|
|
|
|
|
|
|
// These files are considered bottommost because none of their keys can exist
|
|
|
|
// at lower levels. They are not necessarily all in the same level. The marked
|
|
|
|
// ones are eligible for compaction because they contain duplicate key
|
|
|
|
// versions that are no longer protected by snapshot. These variables are
|
|
|
|
// protected by DB mutex and are calculated in `GenerateBottommostFiles()` and
|
|
|
|
// `ComputeBottommostFilesMarkedForCompaction()`.
|
|
|
|
autovector<std::pair<int, FileMetaData*>> bottommost_files_;
|
|
|
|
autovector<std::pair<int, FileMetaData*>>
|
|
|
|
bottommost_files_marked_for_compaction_;
|
|
|
|
|
|
|
|
// Threshold for needing to mark another bottommost file. Maintain it so we
|
|
|
|
// can quickly check when releasing a snapshot whether more bottommost files
|
|
|
|
// became eligible for compaction. It's defined as the min of the max nonzero
|
|
|
|
// seqnums of unmarked bottommost files.
|
|
|
|
SequenceNumber bottommost_files_mark_threshold_ = kMaxSequenceNumber;
|
|
|
|
|
|
|
|
// Monotonically increases as we release old snapshots. Zero indicates no
|
|
|
|
// snapshots have been released yet. When no snapshots remain we set it to the
|
|
|
|
// current seqnum, which needs to be protected as a snapshot can still be
|
|
|
|
// created that references it.
|
|
|
|
SequenceNumber oldest_snapshot_seqnum_ = 0;
|
|
|
|
|
|
|
|
// Level that should be compacted next and its compaction score.
|
|
|
|
// Score < 1 means compaction is not strictly needed. These fields
|
|
|
|
// are initialized by Finalize().
|
|
|
|
// The most critical level to be compacted is listed first
|
|
|
|
// These are used to pick the best compaction level
|
|
|
|
std::vector<double> compaction_score_;
|
|
|
|
std::vector<int> compaction_level_;
|
|
|
|
int l0_delay_trigger_count_ = 0; // Count used to trigger slow down and stop
|
|
|
|
// for number of L0 files.
|
|
|
|
|
|
|
|
// the following are the sampled temporary stats.
|
|
|
|
// the current accumulated size of sampled files.
|
|
|
|
uint64_t accumulated_file_size_;
|
|
|
|
// the current accumulated size of all raw keys based on the sampled files.
|
|
|
|
uint64_t accumulated_raw_key_size_;
|
|
|
|
// the current accumulated size of all raw keys based on the sampled files.
|
|
|
|
uint64_t accumulated_raw_value_size_;
|
|
|
|
// total number of non-deletion entries
|
|
|
|
uint64_t accumulated_num_non_deletions_;
|
|
|
|
// total number of deletion entries
|
|
|
|
uint64_t accumulated_num_deletions_;
|
|
|
|
// current number of non_deletion entries
|
|
|
|
uint64_t current_num_non_deletions_;
|
|
|
|
// current number of deletion entries
|
|
|
|
uint64_t current_num_deletions_;
|
|
|
|
// current number of file samples
|
|
|
|
uint64_t current_num_samples_;
|
|
|
|
// Estimated bytes needed to be compacted until all levels' size is down to
|
|
|
|
// target sizes.
|
|
|
|
uint64_t estimated_compaction_needed_bytes_;
|
|
|
|
|
|
|
|
bool finalized_;
|
|
|
|
|
|
|
|
// If set to true, we will run consistency checks even if RocksDB
|
|
|
|
// is compiled in release mode
|
|
|
|
bool force_consistency_checks_;
|
|
|
|
|
|
|
|
friend class Version;
|
|
|
|
friend class VersionSet;
|
|
|
|
// No copying allowed
|
|
|
|
VersionStorageInfo(const VersionStorageInfo&) = delete;
|
|
|
|
void operator=(const VersionStorageInfo&) = delete;
|
|
|
|
};
|
|
|
|
|
Introduce a new MultiGet batching implementation (#5011)
Summary:
This PR introduces a new MultiGet() API, with the underlying implementation grouping keys based on SST file and batching lookups in a file. The reason for the new API is twofold - the definition allows callers to allocate storage for status and values on stack instead of std::vector, as well as return values as PinnableSlices in order to avoid copying, and it keeps the original MultiGet() implementation intact while we experiment with batching.
Batching is useful when there is some spatial locality to the keys being queries, as well as larger batch sizes. The main benefits are due to -
1. Fewer function calls, especially to BlockBasedTableReader::MultiGet() and FullFilterBlockReader::KeysMayMatch()
2. Bloom filter cachelines can be prefetched, hiding the cache miss latency
The next step is to optimize the binary searches in the level_storage_info, index blocks and data blocks, since we could reduce the number of key comparisons if the keys are relatively close to each other. The batching optimizations also need to be extended to other formats, such as PlainTable and filter formats. This also needs to be added to db_stress.
Benchmark results from db_bench for various batch size/locality of reference combinations are given below. Locality was simulated by offsetting the keys in a batch by a stride length. Each SST file is about 8.6MB uncompressed and key/value size is 16/100 uncompressed. To focus on the cpu benefit of batching, the runs were single threaded and bound to the same cpu to eliminate interference from other system events. The results show a 10-25% improvement in micros/op from smaller to larger batch sizes (4 - 32).
Batch Sizes
1 | 2 | 4 | 8 | 16 | 32
Random pattern (Stride length 0)
4.158 | 4.109 | 4.026 | 4.05 | 4.1 | 4.074 - Get
4.438 | 4.302 | 4.165 | 4.122 | 4.096 | 4.075 - MultiGet (no batching)
4.461 | 4.256 | 4.277 | 4.11 | 4.182 | 4.14 - MultiGet (w/ batching)
Good locality (Stride length 16)
4.048 | 3.659 | 3.248 | 2.99 | 2.84 | 2.753
4.429 | 3.728 | 3.406 | 3.053 | 2.911 | 2.781
4.452 | 3.45 | 2.833 | 2.451 | 2.233 | 2.135
Good locality (Stride length 256)
4.066 | 3.786 | 3.581 | 3.447 | 3.415 | 3.232
4.406 | 4.005 | 3.644 | 3.49 | 3.381 | 3.268
4.393 | 3.649 | 3.186 | 2.882 | 2.676 | 2.62
Medium locality (Stride length 4096)
4.012 | 3.922 | 3.768 | 3.61 | 3.582 | 3.555
4.364 | 4.057 | 3.791 | 3.65 | 3.57 | 3.465
4.479 | 3.758 | 3.316 | 3.077 | 2.959 | 2.891
dbbench command used (on a DB with 4 levels, 12 million keys)-
TEST_TMPDIR=/dev/shm numactl -C 10 ./db_bench.tmp -use_existing_db=true -benchmarks="readseq,multireadrandom" -write_buffer_size=4194304 -target_file_size_base=4194304 -max_bytes_for_level_base=16777216 -num=12000000 -reads=12000000 -duration=90 -threads=1 -compression_type=none -cache_size=4194304000 -batch_size=32 -disable_auto_compactions=true -bloom_bits=10 -cache_index_and_filter_blocks=true -pin_l0_filter_and_index_blocks_in_cache=true -multiread_batched=true -multiread_stride=4
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5011
Differential Revision: D14348703
Pulled By: anand1976
fbshipit-source-id: 774406dab3776d979c809522a67bedac6c17f84b
6 years ago
|
|
|
using MultiGetRange = MultiGetContext::Range;
|
|
|
|
// A column family's version consists of the SST files owned by the column
|
|
|
|
// family at a certain point in time.
|
|
|
|
class Version {
|
|
|
|
public:
|
|
|
|
// Append to *iters a sequence of iterators that will
|
|
|
|
// yield the contents of this Version when merged together.
|
|
|
|
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
|
|
|
void AddIterators(const ReadOptions&, const EnvOptions& soptions,
|
|
|
|
MergeIteratorBuilder* merger_iter_builder,
|
|
|
|
RangeDelAggregator* range_del_agg);
|
|
|
|
|
|
|
|
void AddIteratorsForLevel(const ReadOptions&, const EnvOptions& soptions,
|
|
|
|
MergeIteratorBuilder* merger_iter_builder,
|
|
|
|
int level, RangeDelAggregator* range_del_agg);
|
|
|
|
|
|
|
|
Status OverlapWithLevelIterator(const ReadOptions&, const EnvOptions&,
|
|
|
|
const Slice& smallest_user_key,
|
|
|
|
const Slice& largest_user_key,
|
|
|
|
int level, bool* overlap);
|
|
|
|
|
|
|
|
// Lookup the value for key. If found, store it in *val and
|
|
|
|
// return OK. Else return a non-OK status.
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
9 years ago
|
|
|
// Uses *operands to store merge_operator operations to apply later.
|
|
|
|
//
|
|
|
|
// If the ReadOptions.read_tier is set to do a read-only fetch, then
|
|
|
|
// *value_found will be set to false if it cannot be determined whether
|
|
|
|
// this value exists without doing IO.
|
|
|
|
//
|
|
|
|
// If the key is Deleted, *status will be set to NotFound and
|
|
|
|
// *key_exists will be set to true.
|
|
|
|
// If no key was found, *status will be set to NotFound and
|
|
|
|
// *key_exists will be set to false.
|
|
|
|
// If seq is non-null, *seq will be set to the sequence number found
|
|
|
|
// for the key if a key was found.
|
|
|
|
//
|
|
|
|
// REQUIRES: lock is not held
|
|
|
|
void Get(const ReadOptions&, const LookupKey& key, PinnableSlice* value,
|
|
|
|
Status* status, MergeContext* merge_context,
|
Use only "local" range tombstones during Get (#4449)
Summary:
Previously, range tombstones were accumulated from every level, which
was necessary if a range tombstone in a higher level covered a key in a lower
level. However, RangeDelAggregator::AddTombstones's complexity is based on
the number of tombstones that are currently stored in it, which is wasteful in
the Get case, where we only need to know the highest sequence number of range
tombstones that cover the key from higher levels, and compute the highest covering
sequence number at the current level. This change introduces this optimization, and
removes the use of RangeDelAggregator from the Get path.
In the benchmark results, the following command was used to initialize the database:
```
./db_bench -db=/dev/shm/5k-rts -use_existing_db=false -benchmarks=filluniquerandom -write_buffer_size=1048576 -compression_type=lz4 -target_file_size_base=1048576 -max_bytes_for_level_base=4194304 -value_size=112 -key_size=16 -block_size=4096 -level_compaction_dynamic_level_bytes=true -num=5000000 -max_background_jobs=12 -benchmark_write_rate_limit=20971520 -range_tombstone_width=100 -writes_per_range_tombstone=100 -max_num_range_tombstones=50000 -bloom_bits=8
```
...and the following command was used to measure read throughput:
```
./db_bench -db=/dev/shm/5k-rts/ -use_existing_db=true -benchmarks=readrandom -disable_auto_compactions=true -num=5000000 -reads=100000 -threads=32
```
The filluniquerandom command was only run once, and the resulting database was used
to measure read performance before and after the PR. Both binaries were compiled with
`DEBUG_LEVEL=0`.
Readrandom results before PR:
```
readrandom : 4.544 micros/op 220090 ops/sec; 16.9 MB/s (63103 of 100000 found)
```
Readrandom results after PR:
```
readrandom : 11.147 micros/op 89707 ops/sec; 6.9 MB/s (63103 of 100000 found)
```
So it's actually slower right now, but this PR paves the way for future optimizations (see #4493).
----
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4449
Differential Revision: D10370575
Pulled By: abhimadan
fbshipit-source-id: 9a2e152be1ef36969055c0e9eb4beb0d96c11f4d
6 years ago
|
|
|
SequenceNumber* max_covering_tombstone_seq,
|
|
|
|
bool* value_found = nullptr, bool* key_exists = nullptr,
|
|
|
|
SequenceNumber* seq = nullptr, ReadCallback* callback = nullptr,
|
|
|
|
bool* is_blob = nullptr);
|
|
|
|
|
Introduce a new MultiGet batching implementation (#5011)
Summary:
This PR introduces a new MultiGet() API, with the underlying implementation grouping keys based on SST file and batching lookups in a file. The reason for the new API is twofold - the definition allows callers to allocate storage for status and values on stack instead of std::vector, as well as return values as PinnableSlices in order to avoid copying, and it keeps the original MultiGet() implementation intact while we experiment with batching.
Batching is useful when there is some spatial locality to the keys being queries, as well as larger batch sizes. The main benefits are due to -
1. Fewer function calls, especially to BlockBasedTableReader::MultiGet() and FullFilterBlockReader::KeysMayMatch()
2. Bloom filter cachelines can be prefetched, hiding the cache miss latency
The next step is to optimize the binary searches in the level_storage_info, index blocks and data blocks, since we could reduce the number of key comparisons if the keys are relatively close to each other. The batching optimizations also need to be extended to other formats, such as PlainTable and filter formats. This also needs to be added to db_stress.
Benchmark results from db_bench for various batch size/locality of reference combinations are given below. Locality was simulated by offsetting the keys in a batch by a stride length. Each SST file is about 8.6MB uncompressed and key/value size is 16/100 uncompressed. To focus on the cpu benefit of batching, the runs were single threaded and bound to the same cpu to eliminate interference from other system events. The results show a 10-25% improvement in micros/op from smaller to larger batch sizes (4 - 32).
Batch Sizes
1 | 2 | 4 | 8 | 16 | 32
Random pattern (Stride length 0)
4.158 | 4.109 | 4.026 | 4.05 | 4.1 | 4.074 - Get
4.438 | 4.302 | 4.165 | 4.122 | 4.096 | 4.075 - MultiGet (no batching)
4.461 | 4.256 | 4.277 | 4.11 | 4.182 | 4.14 - MultiGet (w/ batching)
Good locality (Stride length 16)
4.048 | 3.659 | 3.248 | 2.99 | 2.84 | 2.753
4.429 | 3.728 | 3.406 | 3.053 | 2.911 | 2.781
4.452 | 3.45 | 2.833 | 2.451 | 2.233 | 2.135
Good locality (Stride length 256)
4.066 | 3.786 | 3.581 | 3.447 | 3.415 | 3.232
4.406 | 4.005 | 3.644 | 3.49 | 3.381 | 3.268
4.393 | 3.649 | 3.186 | 2.882 | 2.676 | 2.62
Medium locality (Stride length 4096)
4.012 | 3.922 | 3.768 | 3.61 | 3.582 | 3.555
4.364 | 4.057 | 3.791 | 3.65 | 3.57 | 3.465
4.479 | 3.758 | 3.316 | 3.077 | 2.959 | 2.891
dbbench command used (on a DB with 4 levels, 12 million keys)-
TEST_TMPDIR=/dev/shm numactl -C 10 ./db_bench.tmp -use_existing_db=true -benchmarks="readseq,multireadrandom" -write_buffer_size=4194304 -target_file_size_base=4194304 -max_bytes_for_level_base=16777216 -num=12000000 -reads=12000000 -duration=90 -threads=1 -compression_type=none -cache_size=4194304000 -batch_size=32 -disable_auto_compactions=true -bloom_bits=10 -cache_index_and_filter_blocks=true -pin_l0_filter_and_index_blocks_in_cache=true -multiread_batched=true -multiread_stride=4
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5011
Differential Revision: D14348703
Pulled By: anand1976
fbshipit-source-id: 774406dab3776d979c809522a67bedac6c17f84b
6 years ago
|
|
|
void MultiGet(const ReadOptions&, MultiGetRange* range,
|
|
|
|
ReadCallback* callback = nullptr, bool* is_blob = nullptr);
|
|
|
|
|
|
|
|
// Loads some stats information from files. Call without mutex held. It needs
|
|
|
|
// to be called before applying the version to the version set.
|
|
|
|
void PrepareApply(const MutableCFOptions& mutable_cf_options,
|
|
|
|
bool update_stats);
|
|
|
|
|
|
|
|
// Reference count management (so Versions do not disappear out from
|
|
|
|
// under live iterators)
|
|
|
|
void Ref();
|
|
|
|
// Decrease reference count. Delete the object if no reference left
|
|
|
|
// and return true. Otherwise, return false.
|
|
|
|
bool Unref();
|
|
|
|
|
|
|
|
// Add all files listed in the current version to *live.
|
|
|
|
void AddLiveFiles(std::vector<FileDescriptor>* live);
|
|
|
|
|
|
|
|
// Return a human readable string that describes this version's contents.
|
|
|
|
std::string DebugString(bool hex = false, bool print_stats = false) const;
|
|
|
|
|
|
|
|
// Returns the version number of this version
|
|
|
|
uint64_t GetVersionNumber() const { return version_number_; }
|
|
|
|
|
|
|
|
// REQUIRES: lock is held
|
|
|
|
// On success, "tp" will contains the table properties of the file
|
|
|
|
// specified in "file_meta". If the file name of "file_meta" is
|
|
|
|
// known ahead, passing it by a non-null "fname" can save a
|
|
|
|
// file-name conversion.
|
|
|
|
Status GetTableProperties(std::shared_ptr<const TableProperties>* tp,
|
|
|
|
const FileMetaData* file_meta,
|
|
|
|
const std::string* fname = nullptr) const;
|
|
|
|
|
|
|
|
// REQUIRES: lock is held
|
|
|
|
// On success, *props will be populated with all SSTables' table properties.
|
|
|
|
// The keys of `props` are the sst file name, the values of `props` are the
|
|
|
|
// tables' properties, represented as std::shared_ptr.
|
|
|
|
Status GetPropertiesOfAllTables(TablePropertiesCollection* props);
|
|
|
|
Status GetPropertiesOfAllTables(TablePropertiesCollection* props, int level);
|
|
|
|
Status GetPropertiesOfTablesInRange(const Range* range, std::size_t n,
|
|
|
|
TablePropertiesCollection* props) const;
|
|
|
|
|
|
|
|
// REQUIRES: lock is held
|
|
|
|
// On success, "tp" will contains the aggregated table property among
|
|
|
|
// the table properties of all sst files in this version.
|
|
|
|
Status GetAggregatedTableProperties(
|
|
|
|
std::shared_ptr<const TableProperties>* tp, int level = -1);
|
|
|
|
|
|
|
|
uint64_t GetEstimatedActiveKeys() {
|
|
|
|
return storage_info_.GetEstimatedActiveKeys();
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t GetMemoryUsageByTableReaders();
|
|
|
|
|
|
|
|
ColumnFamilyData* cfd() const { return cfd_; }
|
|
|
|
|
|
|
|
// Return the next Version in the linked list. Used for debug only
|
|
|
|
Version* TEST_Next() const {
|
|
|
|
return next_;
|
|
|
|
}
|
|
|
|
|
|
|
|
int TEST_refs() const { return refs_; }
|
|
|
|
|
|
|
|
VersionStorageInfo* storage_info() { return &storage_info_; }
|
|
|
|
|
|
|
|
VersionSet* version_set() { return vset_; }
|
|
|
|
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
10 years ago
|
|
|
void GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta);
|
|
|
|
|
|
|
|
uint64_t GetSstFilesSize();
|
|
|
|
|
|
|
|
MutableCFOptions GetMutableCFOptions() { return mutable_cf_options_; }
|
|
|
|
|
|
|
|
private:
|
|
|
|
Env* env_;
|
|
|
|
friend class ReactiveVersionSet;
|
|
|
|
friend class VersionSet;
|
|
|
|
|
|
|
|
const InternalKeyComparator* internal_comparator() const {
|
|
|
|
return storage_info_.internal_comparator_;
|
|
|
|
}
|
|
|
|
const Comparator* user_comparator() const {
|
|
|
|
return storage_info_.user_comparator_;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool PrefixMayMatch(const ReadOptions& read_options,
|
|
|
|
InternalIterator* level_iter,
|
|
|
|
const Slice& internal_prefix) const;
|
|
|
|
|
Skip bottom-level filter block caching when hit-optimized
Summary:
When Get() or NewIterator() trigger file loads, skip caching the filter block if
(1) optimize_filters_for_hits is set and (2) the file is on the bottommost
level. Also skip checking filters under the same conditions, which means that
for a preloaded file or a file that was trivially-moved to the bottom level, its
filter block will eventually expire from the cache.
- added parameters/instance variables in various places in order to propagate the config ("skip_filters") from version_set to block_based_table_reader
- in BlockBasedTable::Rep, this optimization prevents filter from being loaded when the file is opened simply by setting filter_policy = nullptr
- in BlockBasedTable::Get/BlockBasedTable::NewIterator, this optimization prevents filter from being used (even if it was loaded already) by setting filter = nullptr
Test Plan:
updated unit test:
$ ./db_test --gtest_filter=DBTest.OptimizeFiltersForHits
will also run 'make check'
Reviewers: sdong, igor, paultuckfield, anthony, rven, kradhakrishnan, IslamAbdelRahman, yhchiang
Reviewed By: yhchiang
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D51633
9 years ago
|
|
|
// Returns true if the filter blocks in the specified level will not be
|
|
|
|
// checked during read operations. In certain cases (trivial move or preload),
|
|
|
|
// the filter block may already be cached, but we still do not access it such
|
|
|
|
// that it eventually expires from the cache.
|
|
|
|
bool IsFilterSkipped(int level, bool is_file_last_in_level = false);
|
Skip bottom-level filter block caching when hit-optimized
Summary:
When Get() or NewIterator() trigger file loads, skip caching the filter block if
(1) optimize_filters_for_hits is set and (2) the file is on the bottommost
level. Also skip checking filters under the same conditions, which means that
for a preloaded file or a file that was trivially-moved to the bottom level, its
filter block will eventually expire from the cache.
- added parameters/instance variables in various places in order to propagate the config ("skip_filters") from version_set to block_based_table_reader
- in BlockBasedTable::Rep, this optimization prevents filter from being loaded when the file is opened simply by setting filter_policy = nullptr
- in BlockBasedTable::Get/BlockBasedTable::NewIterator, this optimization prevents filter from being used (even if it was loaded already) by setting filter = nullptr
Test Plan:
updated unit test:
$ ./db_test --gtest_filter=DBTest.OptimizeFiltersForHits
will also run 'make check'
Reviewers: sdong, igor, paultuckfield, anthony, rven, kradhakrishnan, IslamAbdelRahman, yhchiang
Reviewed By: yhchiang
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D51633
9 years ago
|
|
|
|
|
|
|
// The helper function of UpdateAccumulatedStats, which may fill the missing
|
|
|
|
// fields of file_meta from its associated TableProperties.
|
|
|
|
// Returns true if it does initialize FileMetaData.
|
|
|
|
bool MaybeInitializeFileMetaData(FileMetaData* file_meta);
|
|
|
|
|
|
|
|
// Update the accumulated stats associated with the current version.
|
|
|
|
// This accumulated stats will be used in compaction.
|
|
|
|
void UpdateAccumulatedStats(bool update_stats);
|
|
|
|
|
|
|
|
// Sort all files for this version based on their file size and
|
|
|
|
// record results in files_by_compaction_pri_. The largest files are listed
|
|
|
|
// first.
|
|
|
|
void UpdateFilesByCompactionPri();
|
|
|
|
|
|
|
|
ColumnFamilyData* cfd_; // ColumnFamilyData to which this Version belongs
|
|
|
|
Logger* info_log_;
|
|
|
|
Statistics* db_statistics_;
|
|
|
|
TableCache* table_cache_;
|
|
|
|
const MergeOperator* merge_operator_;
|
create compressed_levels_ in Version, allocate its space using arena. Make Version::Get, Version::FindFile faster
Summary:
Define CompressedFileMetaData that just contains fd, smallest_slice, largest_slice. Create compressed_levels_ in Version, the space is allocated using arena
Thus increase the file meta data locality, speed up "Get" and "FindFile"
benchmark with in-memory tmpfs, could have 4% improvement under "random read" and 2% improvement under "read while writing"
benchmark command:
./db_bench --db=/mnt/db/rocksdb --num_levels=6 --key_size=20 --prefix_size=20 --keys_per_prefix=0 --value_size=100 --block_size=4096 --cache_size=17179869184 --cache_numshardbits=6 --compression_type=none --compression_ratio=1 --min_level_to_compress=-1 --disable_seek_compaction=1 --hard_rate_limit=2 --write_buffer_size=134217728 --max_write_buffer_number=2 --level0_file_num_compaction_trigger=8 --target_file_size_base=33554432 --max_bytes_for_level_base=1073741824 --disable_wal=0 --sync=0 --disable_data_sync=1 --verify_checksum=1 --delete_obsolete_files_period_micros=314572800 --max_grandparent_overlap_factor=10 --max_background_compactions=4 --max_background_flushes=0 --level0_slowdown_writes_trigger=16 --level0_stop_writes_trigger=24 --statistics=0 --stats_per_interval=0 --stats_interval=1048576 --histogram=0 --use_plain_table=1 --open_files=-1 --mmap_read=1 --mmap_write=0 --memtablerep=prefix_hash --bloom_bits=10 --bloom_locality=1 --perf_level=0 --benchmarks=readwhilewriting,readwhilewriting,readwhilewriting --use_existing_db=1 --num=52428800 --threads=1 —writes_per_second=81920
Read Random:
From 1.8363 ms/op, improve to 1.7587 ms/op.
Read while writing:
From 2.985 ms/op, improve to 2.924 ms/op.
Test Plan:
make all check
Reviewers: ljin, haobo, yhchiang, sdong
Reviewed By: sdong
Subscribers: dhruba, igor
Differential Revision: https://reviews.facebook.net/D19419
11 years ago
|
|
|
|
|
|
|
VersionStorageInfo storage_info_;
|
|
|
|
VersionSet* vset_; // VersionSet to which this Version belongs
|
|
|
|
Version* next_; // Next version in linked list
|
|
|
|
Version* prev_; // Previous version in linked list
|
|
|
|
int refs_; // Number of live refs to this version
|
|
|
|
const EnvOptions env_options_;
|
|
|
|
const MutableCFOptions mutable_cf_options_;
|
|
|
|
|
|
|
|
// A version number that uniquely represents this version. This is
|
|
|
|
// used for debugging and logging purposes only.
|
|
|
|
uint64_t version_number_;
|
|
|
|
|
|
|
|
Version(ColumnFamilyData* cfd, VersionSet* vset, const EnvOptions& env_opt,
|
|
|
|
MutableCFOptions mutable_cf_options, uint64_t version_number = 0);
|
|
|
|
|
|
|
|
~Version();
|
|
|
|
|
|
|
|
// No copying allowed
|
|
|
|
Version(const Version&);
|
|
|
|
void operator=(const Version&);
|
|
|
|
};
|
|
|
|
|
|
|
|
struct ObsoleteFileInfo {
|
|
|
|
FileMetaData* metadata;
|
|
|
|
std::string path;
|
|
|
|
|
|
|
|
ObsoleteFileInfo() noexcept : metadata(nullptr) {}
|
|
|
|
ObsoleteFileInfo(FileMetaData* f, const std::string& file_path)
|
|
|
|
: metadata(f), path(file_path) {}
|
|
|
|
|
|
|
|
ObsoleteFileInfo(const ObsoleteFileInfo&) = delete;
|
|
|
|
ObsoleteFileInfo& operator=(const ObsoleteFileInfo&) = delete;
|
|
|
|
|
|
|
|
ObsoleteFileInfo(ObsoleteFileInfo&& rhs) noexcept :
|
|
|
|
ObsoleteFileInfo() {
|
|
|
|
*this = std::move(rhs);
|
|
|
|
}
|
|
|
|
|
|
|
|
ObsoleteFileInfo& operator=(ObsoleteFileInfo&& rhs) noexcept {
|
|
|
|
path = std::move(rhs.path);
|
|
|
|
metadata = rhs.metadata;
|
|
|
|
rhs.metadata = nullptr;
|
|
|
|
|
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
|
|
|
|
void DeleteMetadata() {
|
|
|
|
delete metadata;
|
|
|
|
metadata = nullptr;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
class BaseReferencedVersionBuilder;
|
|
|
|
|
|
|
|
class AtomicGroupReadBuffer {
|
|
|
|
public:
|
|
|
|
Status AddEdit(VersionEdit* edit);
|
|
|
|
void Clear();
|
|
|
|
bool IsFull() const;
|
|
|
|
bool IsEmpty() const;
|
|
|
|
|
|
|
|
uint64_t TEST_read_edits_in_atomic_group() const {
|
|
|
|
return read_edits_in_atomic_group_;
|
|
|
|
}
|
|
|
|
std::vector<VersionEdit>& replay_buffer() { return replay_buffer_; }
|
|
|
|
|
|
|
|
private:
|
|
|
|
uint64_t read_edits_in_atomic_group_ = 0;
|
|
|
|
std::vector<VersionEdit> replay_buffer_;
|
|
|
|
};
|
|
|
|
|
|
|
|
// VersionSet is the collection of versions of all the column families of the
|
|
|
|
// database. Each database owns one VersionSet. A VersionSet has access to all
|
|
|
|
// column families via ColumnFamilySet, i.e. set of the column families.
|
|
|
|
class VersionSet {
|
|
|
|
public:
|
|
|
|
VersionSet(const std::string& dbname, const ImmutableDBOptions* db_options,
|
|
|
|
const EnvOptions& env_options, Cache* table_cache,
|
|
|
|
WriteBufferManager* write_buffer_manager,
|
|
|
|
WriteController* write_controller);
|
|
|
|
virtual ~VersionSet();
|
|
|
|
|
|
|
|
// Apply *edit to the current version to form a new descriptor that
|
|
|
|
// is both saved to persistent state and installed as the new
|
|
|
|
// current version. Will release *mu while actually writing to the file.
|
|
|
|
// column_family_options has to be set if edit is column family add
|
|
|
|
// REQUIRES: *mu is held on entry.
|
|
|
|
// REQUIRES: no other thread concurrently calls LogAndApply()
|
|
|
|
Status LogAndApply(
|
|
|
|
ColumnFamilyData* column_family_data,
|
|
|
|
const MutableCFOptions& mutable_cf_options, VersionEdit* edit,
|
|
|
|
InstrumentedMutex* mu, Directory* db_directory = nullptr,
|
|
|
|
bool new_descriptor_log = false,
|
|
|
|
const ColumnFamilyOptions* column_family_options = nullptr) {
|
|
|
|
autovector<ColumnFamilyData*> cfds;
|
|
|
|
cfds.emplace_back(column_family_data);
|
|
|
|
autovector<const MutableCFOptions*> mutable_cf_options_list;
|
|
|
|
mutable_cf_options_list.emplace_back(&mutable_cf_options);
|
|
|
|
autovector<autovector<VersionEdit*>> edit_lists;
|
|
|
|
autovector<VersionEdit*> edit_list;
|
|
|
|
edit_list.emplace_back(edit);
|
|
|
|
edit_lists.emplace_back(edit_list);
|
|
|
|
return LogAndApply(cfds, mutable_cf_options_list, edit_lists, mu,
|
|
|
|
db_directory, new_descriptor_log, column_family_options);
|
|
|
|
}
|
|
|
|
// The batch version. If edit_list.size() > 1, caller must ensure that
|
|
|
|
// no edit in the list column family add or drop
|
|
|
|
Status LogAndApply(
|
|
|
|
ColumnFamilyData* column_family_data,
|
|
|
|
const MutableCFOptions& mutable_cf_options,
|
|
|
|
const autovector<VersionEdit*>& edit_list, InstrumentedMutex* mu,
|
|
|
|
Directory* db_directory = nullptr, bool new_descriptor_log = false,
|
|
|
|
const ColumnFamilyOptions* column_family_options = nullptr) {
|
|
|
|
autovector<ColumnFamilyData*> cfds;
|
|
|
|
cfds.emplace_back(column_family_data);
|
|
|
|
autovector<const MutableCFOptions*> mutable_cf_options_list;
|
|
|
|
mutable_cf_options_list.emplace_back(&mutable_cf_options);
|
|
|
|
autovector<autovector<VersionEdit*>> edit_lists;
|
|
|
|
edit_lists.emplace_back(edit_list);
|
|
|
|
return LogAndApply(cfds, mutable_cf_options_list, edit_lists, mu,
|
|
|
|
db_directory, new_descriptor_log, column_family_options);
|
|
|
|
}
|
|
|
|
|
|
|
|
// The across-multi-cf batch version. If edit_lists contain more than
|
|
|
|
// 1 version edits, caller must ensure that no edit in the []list is column
|
|
|
|
// family manipulation.
|
|
|
|
virtual Status LogAndApply(
|
|
|
|
const autovector<ColumnFamilyData*>& cfds,
|
|
|
|
const autovector<const MutableCFOptions*>& mutable_cf_options_list,
|
|
|
|
const autovector<autovector<VersionEdit*>>& edit_lists,
|
|
|
|
InstrumentedMutex* mu, Directory* db_directory = nullptr,
|
|
|
|
bool new_descriptor_log = false,
|
|
|
|
const ColumnFamilyOptions* new_cf_options = nullptr);
|
|
|
|
|
|
|
|
static Status GetCurrentManifestPath(const std::string& dbname, Env* env,
|
|
|
|
std::string* manifest_filename,
|
|
|
|
uint64_t* manifest_file_number);
|
|
|
|
|
|
|
|
// Recover the last saved descriptor from persistent storage.
|
|
|
|
// If read_only == true, Recover() will not complain if some column families
|
|
|
|
// are not opened
|
|
|
|
Status Recover(const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
|
|
bool read_only = false);
|
|
|
|
|
|
|
|
// Reads a manifest file and returns a list of column families in
|
|
|
|
// column_families.
|
|
|
|
static Status ListColumnFamilies(std::vector<std::string>* column_families,
|
|
|
|
const std::string& dbname, Env* env);
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
// Try to reduce the number of levels. This call is valid when
|
|
|
|
// only one level from the new max level to the old
|
|
|
|
// max level containing files.
|
Make VersionSet::ReduceNumberOfLevels() static
Summary:
A lot of our code implicitly assumes number_levels to be static. ReduceNumberOfLevels() breaks that assumption. For example, after calling ReduceNumberOfLevels(), DBImpl::NumberLevels() will be different from VersionSet::NumberLevels(). This is dangerous. Thankfully, it's not in public headers and is only used from LDB cmd tool. LDB tool is only using it statically, i.e. it never calls it with running DB instance. With this diff, we make it explicitly static. This way, we can assume number_levels to be immutable and not break assumption that lot of our code is relying upon. LDB tool can still use the method.
Also, I removed the method from a separate file since it breaks filename completition. version_se<TAB> now completes to "version_set." instead of "version_set" (without the dot). I don't see a big reason that the function should be in a different file.
Test Plan: reduce_levels_test
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15303
11 years ago
|
|
|
// The call is static, since number of levels is immutable during
|
|
|
|
// the lifetime of a RocksDB instance. It reduces number of levels
|
|
|
|
// in a DB by applying changes to manifest.
|
|
|
|
// For example, a db currently has 7 levels [0-6], and a call to
|
|
|
|
// to reduce to 5 [0-4] can only be executed when only one level
|
|
|
|
// among [4-6] contains files.
|
Make VersionSet::ReduceNumberOfLevels() static
Summary:
A lot of our code implicitly assumes number_levels to be static. ReduceNumberOfLevels() breaks that assumption. For example, after calling ReduceNumberOfLevels(), DBImpl::NumberLevels() will be different from VersionSet::NumberLevels(). This is dangerous. Thankfully, it's not in public headers and is only used from LDB cmd tool. LDB tool is only using it statically, i.e. it never calls it with running DB instance. With this diff, we make it explicitly static. This way, we can assume number_levels to be immutable and not break assumption that lot of our code is relying upon. LDB tool can still use the method.
Also, I removed the method from a separate file since it breaks filename completition. version_se<TAB> now completes to "version_set." instead of "version_set" (without the dot). I don't see a big reason that the function should be in a different file.
Test Plan: reduce_levels_test
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15303
11 years ago
|
|
|
static Status ReduceNumberOfLevels(const std::string& dbname,
|
|
|
|
const Options* options,
|
|
|
|
const EnvOptions& env_options,
|
Make VersionSet::ReduceNumberOfLevels() static
Summary:
A lot of our code implicitly assumes number_levels to be static. ReduceNumberOfLevels() breaks that assumption. For example, after calling ReduceNumberOfLevels(), DBImpl::NumberLevels() will be different from VersionSet::NumberLevels(). This is dangerous. Thankfully, it's not in public headers and is only used from LDB cmd tool. LDB tool is only using it statically, i.e. it never calls it with running DB instance. With this diff, we make it explicitly static. This way, we can assume number_levels to be immutable and not break assumption that lot of our code is relying upon. LDB tool can still use the method.
Also, I removed the method from a separate file since it breaks filename completition. version_se<TAB> now completes to "version_set." instead of "version_set" (without the dot). I don't see a big reason that the function should be in a different file.
Test Plan: reduce_levels_test
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15303
11 years ago
|
|
|
int new_levels);
|
|
|
|
|
|
|
|
// printf contents (for debugging)
|
|
|
|
Status DumpManifest(Options& options, std::string& manifestFileName,
|
Added JSON manifest dump option to ldb command
Summary:
Added a new flag --json to the ldb manifest_dump command
that prints out the version edits as JSON objects for easier
reading and parsing of information.
Test Plan:
**Sample usage: **
```
./ldb manifest_dump --json --path=path/to/manifest/file
```
**Sample output:**
```
{"EditNumber": 0, "Comparator": "leveldb.BytewiseComparator", "ColumnFamily": 0}
{"EditNumber": 1, "LogNumber": 0, "ColumnFamily": 0}
{"EditNumber": 2, "LogNumber": 4, "PrevLogNumber": 0, "NextFileNumber": 7, "LastSeq": 35356, "AddedFiles": [{"Level": 0, "FileNumber": 5, "FileSize": 1949284, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
{"EditNumber": 13, "PrevLogNumber": 0, "NextFileNumber": 36, "LastSeq": 290994, "DeletedFiles": [{"Level": 0, "FileNumber": 17}, {"Level": 0, "FileNumber": 20}, {"Level": 0, "FileNumber": 22}, {"Level": 0, "FileNumber": 24}, {"Level": 1, "FileNumber": 13}, {"Level": 1, "FileNumber": 14}, {"Level": 1, "FileNumber": 15}, {"Level": 1, "FileNumber": 18}], "AddedFiles": [{"Level": 1, "FileNumber": 25, "FileSize": 2114340, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 26, "FileSize": 2115213, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 27, "FileSize": 2114807, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 30, "FileSize": 2115271, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 31, "FileSize": 2115165, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 32, "FileSize": 2114683, "SmallestIKey": "'", "LargestIKey": "'"}, {"Level": 1, "FileNumber": 35, "FileSize": 1757512, "SmallestIKey": "'", "LargestIKey": "'"}], "ColumnFamily": 0}
...
```
Reviewers: sdong, anthony, yhchiang, igor
Reviewed By: igor
Subscribers: dhruba
Differential Revision: https://reviews.facebook.net/D41727
10 years ago
|
|
|
bool verbose, bool hex = false, bool json = false);
|
|
|
|
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
|
|
|
// Return the current manifest file number
|
|
|
|
uint64_t manifest_file_number() const { return manifest_file_number_; }
|
|
|
|
|
|
|
|
uint64_t options_file_number() const { return options_file_number_; }
|
|
|
|
|
|
|
|
uint64_t pending_manifest_file_number() const {
|
|
|
|
return pending_manifest_file_number_;
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t current_next_file_number() const { return next_file_number_.load(); }
|
|
|
|
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
uint64_t min_log_number_to_keep_2pc() const {
|
|
|
|
return min_log_number_to_keep_2pc_.load();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Allocate and return a new file number
|
|
|
|
uint64_t NewFileNumber() { return next_file_number_.fetch_add(1); }
|
|
|
|
|
|
|
|
// Fetch And Add n new file number
|
|
|
|
uint64_t FetchAddFileNumber(uint64_t n) {
|
|
|
|
return next_file_number_.fetch_add(n);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return the last sequence number.
|
|
|
|
uint64_t LastSequence() const {
|
|
|
|
return last_sequence_.load(std::memory_order_acquire);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Note: memory_order_acquire must be sufficient.
|
|
|
|
uint64_t LastAllocatedSequence() const {
|
|
|
|
return last_allocated_sequence_.load(std::memory_order_seq_cst);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Note: memory_order_acquire must be sufficient.
|
|
|
|
uint64_t LastPublishedSequence() const {
|
|
|
|
return last_published_sequence_.load(std::memory_order_seq_cst);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set the last sequence number to s.
|
|
|
|
void SetLastSequence(uint64_t s) {
|
|
|
|
assert(s >= last_sequence_);
|
|
|
|
// Last visible sequence must always be less than last written seq
|
|
|
|
assert(!db_options_->two_write_queues || s <= last_allocated_sequence_);
|
|
|
|
last_sequence_.store(s, std::memory_order_release);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Note: memory_order_release must be sufficient
|
|
|
|
void SetLastPublishedSequence(uint64_t s) {
|
|
|
|
assert(s >= last_published_sequence_);
|
|
|
|
last_published_sequence_.store(s, std::memory_order_seq_cst);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Note: memory_order_release must be sufficient
|
|
|
|
void SetLastAllocatedSequence(uint64_t s) {
|
|
|
|
assert(s >= last_allocated_sequence_);
|
|
|
|
last_allocated_sequence_.store(s, std::memory_order_seq_cst);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Note: memory_order_release must be sufficient
|
|
|
|
uint64_t FetchAddLastAllocatedSequence(uint64_t s) {
|
|
|
|
return last_allocated_sequence_.fetch_add(s, std::memory_order_seq_cst);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Mark the specified file number as used.
|
|
|
|
// REQUIRED: this is only called during single-threaded recovery or repair.
|
|
|
|
void MarkFileNumberUsed(uint64_t number);
|
|
|
|
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
// Mark the specified log number as deleted
|
|
|
|
// REQUIRED: this is only called during single-threaded recovery or repair, or
|
|
|
|
// from ::LogAndApply where the global mutex is held.
|
|
|
|
void MarkMinLogNumberToKeep2PC(uint64_t number);
|
|
|
|
|
|
|
|
// Return the log file number for the log file that is currently
|
|
|
|
// being compacted, or zero if there is no such log file.
|
|
|
|
uint64_t prev_log_number() const { return prev_log_number_; }
|
|
|
|
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
// Returns the minimum log number which still has data not flushed to any SST
|
|
|
|
// file.
|
|
|
|
// In non-2PC mode, all the log numbers smaller than this number can be safely
|
|
|
|
// deleted.
|
|
|
|
uint64_t MinLogNumberWithUnflushedData() const {
|
|
|
|
return PreComputeMinLogNumberWithUnflushedData(nullptr);
|
|
|
|
}
|
|
|
|
// Returns the minimum log number which still has data not flushed to any SST
|
|
|
|
// file, except data from `cfd_to_skip`.
|
|
|
|
uint64_t PreComputeMinLogNumberWithUnflushedData(
|
|
|
|
const ColumnFamilyData* cfd_to_skip) const {
|
|
|
|
uint64_t min_log_num = std::numeric_limits<uint64_t>::max();
|
|
|
|
for (auto cfd : *column_family_set_) {
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
if (cfd == cfd_to_skip) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
// It's safe to ignore dropped column families here:
|
|
|
|
// cfd->IsDropped() becomes true after the drop is persisted in MANIFEST.
|
|
|
|
if (min_log_num > cfd->GetLogNumber() && !cfd->IsDropped()) {
|
|
|
|
min_log_num = cfd->GetLogNumber();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return min_log_num;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create an iterator that reads over the compaction inputs for "*c".
|
|
|
|
// The caller should delete the iterator when no longer needed.
|
|
|
|
InternalIterator* MakeInputIterator(
|
|
|
|
const Compaction* c, RangeDelAggregator* range_del_agg,
|
|
|
|
const EnvOptions& env_options_compactions);
|
|
|
|
|
|
|
|
// Add all files listed in any live version to *live.
|
|
|
|
void AddLiveFiles(std::vector<FileDescriptor>* live_list);
|
|
|
|
|
|
|
|
// Return the approximate size of data to be scanned for range [start, end)
|
|
|
|
// in levels [start_level, end_level). If end_level == 0 it will search
|
|
|
|
// through all non-empty levels
|
|
|
|
uint64_t ApproximateSize(Version* v, const Slice& start, const Slice& end,
|
|
|
|
int start_level, int end_level, bool for_compaction);
|
|
|
|
|
|
|
|
// Return the size of the current manifest file
|
|
|
|
uint64_t manifest_file_size() const { return manifest_file_size_; }
|
|
|
|
|
|
|
|
// verify that the files that we started with for a compaction
|
|
|
|
// still exist in the current version and in the same original level.
|
|
|
|
// This ensures that a concurrent compaction did not erroneously
|
|
|
|
// pick the same files to compact.
|
|
|
|
bool VerifyCompactionFileConsistency(Compaction* c);
|
|
|
|
|
|
|
|
Status GetMetadataForFile(uint64_t number, int* filelevel,
|
|
|
|
FileMetaData** metadata, ColumnFamilyData** cfd);
|
|
|
|
|
|
|
|
// This function doesn't support leveldb SST filenames
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
10 years ago
|
|
|
void GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata);
|
|
|
|
|
|
|
|
void GetObsoleteFiles(std::vector<ObsoleteFileInfo>* files,
|
|
|
|
std::vector<std::string>* manifest_filenames,
|
|
|
|
uint64_t min_pending_output);
|
|
|
|
|
|
|
|
ColumnFamilySet* GetColumnFamilySet() { return column_family_set_.get(); }
|
|
|
|
const EnvOptions& env_options() { return env_options_; }
|
|
|
|
void ChangeEnvOptions(const MutableDBOptions& new_options) {
|
|
|
|
env_options_.writable_file_max_buffer_size =
|
|
|
|
new_options.writable_file_max_buffer_size;
|
|
|
|
}
|
|
|
|
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
const ImmutableDBOptions* db_options() const { return db_options_; }
|
|
|
|
|
|
|
|
static uint64_t GetNumLiveVersions(Version* dummy_versions);
|
|
|
|
|
|
|
|
static uint64_t GetTotalSstFilesSize(Version* dummy_versions);
|
|
|
|
|
|
|
|
protected:
|
|
|
|
struct ManifestWriter;
|
|
|
|
|
|
|
|
friend class Version;
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
10 years ago
|
|
|
friend class DBImpl;
|
|
|
|
friend class DBImplReadOnly;
|
|
|
|
|
|
|
|
struct LogReporter : public log::Reader::Reporter {
|
|
|
|
Status* status;
|
|
|
|
virtual void Corruption(size_t /*bytes*/, const Status& s) override {
|
|
|
|
if (this->status->ok()) *this->status = s;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
// ApproximateSize helper
|
|
|
|
uint64_t ApproximateSizeLevel0(Version* v, const LevelFilesBrief& files_brief,
|
|
|
|
const Slice& start, const Slice& end,
|
|
|
|
bool for_compaction);
|
|
|
|
|
|
|
|
uint64_t ApproximateSize(Version* v, const FdWithKeyRange& f,
|
|
|
|
const Slice& key, bool for_compaction);
|
|
|
|
|
|
|
|
// Save current contents to *log
|
|
|
|
Status WriteSnapshot(log::Writer* log);
|
|
|
|
|
|
|
|
void AppendVersion(ColumnFamilyData* column_family_data, Version* v);
|
|
|
|
|
|
|
|
ColumnFamilyData* CreateColumnFamily(const ColumnFamilyOptions& cf_options,
|
|
|
|
VersionEdit* edit);
|
|
|
|
|
|
|
|
Status ReadAndRecover(
|
|
|
|
log::Reader* reader, AtomicGroupReadBuffer* read_buffer,
|
|
|
|
const std::unordered_map<std::string, ColumnFamilyOptions>&
|
|
|
|
name_to_options,
|
|
|
|
std::unordered_map<int, std::string>& column_families_not_found,
|
|
|
|
std::unordered_map<
|
|
|
|
uint32_t, std::unique_ptr<BaseReferencedVersionBuilder>>& builders,
|
|
|
|
bool* have_log_number, uint64_t* log_number, bool* have_prev_log_number,
|
|
|
|
uint64_t* previous_log_number, bool* have_next_file, uint64_t* next_file,
|
|
|
|
bool* have_last_sequence, SequenceNumber* last_sequence,
|
|
|
|
uint64_t* min_log_number_to_keep, uint32_t* max_column_family);
|
|
|
|
|
|
|
|
// REQUIRES db mutex
|
|
|
|
Status ApplyOneVersionEditToBuilder(
|
|
|
|
VersionEdit& edit,
|
|
|
|
const std::unordered_map<std::string, ColumnFamilyOptions>& name_to_opts,
|
|
|
|
std::unordered_map<int, std::string>& column_families_not_found,
|
|
|
|
std::unordered_map<
|
|
|
|
uint32_t, std::unique_ptr<BaseReferencedVersionBuilder>>& builders,
|
|
|
|
bool* have_log_number, uint64_t* log_number, bool* have_prev_log_number,
|
|
|
|
uint64_t* previous_log_number, bool* have_next_file, uint64_t* next_file,
|
|
|
|
bool* have_last_sequence, SequenceNumber* last_sequence,
|
|
|
|
uint64_t* min_log_number_to_keep, uint32_t* max_column_family);
|
|
|
|
|
|
|
|
Status ExtractInfoFromVersionEdit(
|
|
|
|
ColumnFamilyData* cfd, const VersionEdit& edit, bool* have_log_number,
|
|
|
|
uint64_t* log_number, bool* have_prev_log_number,
|
|
|
|
uint64_t* previous_log_number, bool* have_next_file, uint64_t* next_file,
|
|
|
|
bool* have_last_sequence, SequenceNumber* last_sequence,
|
|
|
|
uint64_t* min_log_number_to_keep, uint32_t* max_column_family);
|
|
|
|
|
|
|
|
std::unique_ptr<ColumnFamilySet> column_family_set_;
|
|
|
|
|
|
|
|
Env* const env_;
|
|
|
|
const std::string dbname_;
|
|
|
|
const ImmutableDBOptions* const db_options_;
|
|
|
|
std::atomic<uint64_t> next_file_number_;
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
// Any log number equal or lower than this should be ignored during recovery,
|
|
|
|
// and is qualified for being deleted in 2PC mode. In non-2PC mode, this
|
|
|
|
// number is ignored.
|
|
|
|
std::atomic<uint64_t> min_log_number_to_keep_2pc_ = {0};
|
|
|
|
uint64_t manifest_file_number_;
|
|
|
|
uint64_t options_file_number_;
|
|
|
|
uint64_t pending_manifest_file_number_;
|
|
|
|
// The last seq visible to reads. It normally indicates the last sequence in
|
|
|
|
// the memtable but when using two write queues it could also indicate the
|
|
|
|
// last sequence in the WAL visible to reads.
|
|
|
|
std::atomic<uint64_t> last_sequence_;
|
|
|
|
// The last seq that is already allocated. It is applicable only when we have
|
|
|
|
// two write queues. In that case seq might or might not have appreated in
|
|
|
|
// memtable but it is expected to appear in the WAL.
|
|
|
|
// We have last_sequence <= last_allocated_sequence_
|
|
|
|
std::atomic<uint64_t> last_allocated_sequence_;
|
|
|
|
// The last allocated sequence that is also published to the readers. This is
|
|
|
|
// applicable only when last_seq_same_as_publish_seq_ is not set. Otherwise
|
|
|
|
// last_sequence_ also indicates the last published seq.
|
|
|
|
// We have last_sequence <= last_published_sequence_ <=
|
|
|
|
// last_allocated_sequence_
|
|
|
|
std::atomic<uint64_t> last_published_sequence_;
|
|
|
|
uint64_t prev_log_number_; // 0 or backing store for memtable being compacted
|
|
|
|
|
|
|
|
// Opened lazily
|
|
|
|
std::unique_ptr<log::Writer> descriptor_log_;
|
|
|
|
|
|
|
|
// generates a increasing version number for every new version
|
|
|
|
uint64_t current_version_number_;
|
|
|
|
|
|
|
|
// Queue of writers to the manifest file
|
|
|
|
std::deque<ManifestWriter*> manifest_writers_;
|
|
|
|
|
|
|
|
// Current size of manifest file
|
|
|
|
uint64_t manifest_file_size_;
|
|
|
|
|
|
|
|
std::vector<ObsoleteFileInfo> obsolete_files_;
|
|
|
|
std::vector<std::string> obsolete_manifests_;
|
|
|
|
|
|
|
|
// env options for all reads and writes except compactions
|
|
|
|
EnvOptions env_options_;
|
|
|
|
|
|
|
|
private:
|
|
|
|
// No copying allowed
|
|
|
|
VersionSet(const VersionSet&);
|
|
|
|
void operator=(const VersionSet&);
|
|
|
|
|
|
|
|
// REQUIRES db mutex at beginning. may release and re-acquire db mutex
|
|
|
|
Status ProcessManifestWrites(std::deque<ManifestWriter>& writers,
|
|
|
|
InstrumentedMutex* mu, Directory* db_directory,
|
|
|
|
bool new_descriptor_log,
|
|
|
|
const ColumnFamilyOptions* new_cf_options);
|
|
|
|
|
|
|
|
void LogAndApplyCFHelper(VersionEdit* edit);
|
|
|
|
void LogAndApplyHelper(ColumnFamilyData* cfd, VersionBuilder* b,
|
|
|
|
VersionEdit* edit, InstrumentedMutex* mu);
|
|
|
|
};
|
|
|
|
|
|
|
|
// ReactiveVersionSet represents a collection of versions of the column
|
|
|
|
// families of the database. Users of ReactiveVersionSet, e.g. DBImplSecondary,
|
|
|
|
// need to replay the MANIFEST (description log in older terms) in order to
|
|
|
|
// reconstruct and install versions.
|
|
|
|
class ReactiveVersionSet : public VersionSet {
|
|
|
|
public:
|
|
|
|
ReactiveVersionSet(const std::string& dbname,
|
|
|
|
const ImmutableDBOptions* _db_options,
|
|
|
|
const EnvOptions& _env_options, Cache* table_cache,
|
|
|
|
WriteBufferManager* write_buffer_manager,
|
|
|
|
WriteController* write_controller);
|
|
|
|
|
|
|
|
~ReactiveVersionSet() override;
|
|
|
|
|
|
|
|
Status ReadAndApply(
|
|
|
|
InstrumentedMutex* mu,
|
|
|
|
std::unique_ptr<log::FragmentBufferedReader>* manifest_reader,
|
|
|
|
std::unordered_set<ColumnFamilyData*>* cfds_changed);
|
|
|
|
|
|
|
|
Status Recover(const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
|
|
std::unique_ptr<log::FragmentBufferedReader>* manifest_reader,
|
|
|
|
std::unique_ptr<log::Reader::Reporter>* manifest_reporter,
|
|
|
|
std::unique_ptr<Status>* manifest_reader_status);
|
|
|
|
|
|
|
|
uint64_t TEST_read_edits_in_atomic_group() const {
|
|
|
|
return read_buffer_.TEST_read_edits_in_atomic_group();
|
|
|
|
}
|
|
|
|
std::vector<VersionEdit>& replay_buffer() {
|
|
|
|
return read_buffer_.replay_buffer();
|
|
|
|
}
|
|
|
|
|
|
|
|
protected:
|
|
|
|
using VersionSet::ApplyOneVersionEditToBuilder;
|
|
|
|
|
|
|
|
// REQUIRES db mutex
|
|
|
|
Status ApplyOneVersionEditToBuilder(
|
|
|
|
VersionEdit& edit, std::unordered_set<ColumnFamilyData*>* cfds_changed,
|
|
|
|
bool* have_log_number, uint64_t* log_number, bool* have_prev_log_number,
|
|
|
|
uint64_t* previous_log_number, bool* have_next_file, uint64_t* next_file,
|
|
|
|
bool* have_last_sequence, SequenceNumber* last_sequence,
|
|
|
|
uint64_t* min_log_number_to_keep, uint32_t* max_column_family);
|
|
|
|
|
|
|
|
Status MaybeSwitchManifest(
|
|
|
|
log::Reader::Reporter* reporter,
|
|
|
|
std::unique_ptr<log::FragmentBufferedReader>* manifest_reader);
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::unordered_map<uint32_t, std::unique_ptr<BaseReferencedVersionBuilder>>
|
|
|
|
active_version_builders_;
|
|
|
|
AtomicGroupReadBuffer read_buffer_;
|
|
|
|
|
|
|
|
using VersionSet::LogAndApply;
|
|
|
|
using VersionSet::Recover;
|
|
|
|
|
|
|
|
Status LogAndApply(
|
|
|
|
const autovector<ColumnFamilyData*>& /*cfds*/,
|
|
|
|
const autovector<const MutableCFOptions*>& /*mutable_cf_options_list*/,
|
|
|
|
const autovector<autovector<VersionEdit*>>& /*edit_lists*/,
|
|
|
|
InstrumentedMutex* /*mu*/, Directory* /*db_directory*/,
|
|
|
|
bool /*new_descriptor_log*/,
|
|
|
|
const ColumnFamilyOptions* /*new_cf_option*/) override {
|
|
|
|
return Status::NotSupported("not supported in reactive mode");
|
|
|
|
}
|
|
|
|
|
|
|
|
// No copy allowed
|
|
|
|
ReactiveVersionSet(const ReactiveVersionSet&);
|
|
|
|
ReactiveVersionSet& operator=(const ReactiveVersionSet&);
|
|
|
|
};
|
|
|
|
|
|
|
|
} // namespace rocksdb
|