|
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
|
//
|
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
#pragma once
|
|
|
|
|
|
|
|
|
|
#include <atomic>
|
|
|
|
|
#include <deque>
|
|
|
|
|
#include <limits>
|
|
|
|
|
#include <set>
|
|
|
|
|
#include <utility>
|
|
|
|
|
#include <vector>
|
|
|
|
|
#include <string>
|
|
|
|
|
|
|
|
|
|
#include "db/column_family.h"
|
|
|
|
|
#include "db/dbformat.h"
|
|
|
|
|
#include "db/flush_scheduler.h"
|
|
|
|
|
#include "db/internal_stats.h"
|
|
|
|
|
#include "db/job_context.h"
|
|
|
|
|
#include "db/log_writer.h"
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
#include "db/logs_with_prep_tracker.h"
|
|
|
|
|
#include "db/memtable_list.h"
|
|
|
|
|
#include "db/snapshot_impl.h"
|
|
|
|
|
#include "db/version_edit.h"
|
|
|
|
|
#include "db/write_controller.h"
|
|
|
|
|
#include "db/write_thread.h"
|
|
|
|
|
#include "monitoring/instrumented_mutex.h"
|
|
|
|
|
#include "options/db_options.h"
|
|
|
|
|
#include "port/port.h"
|
|
|
|
|
#include "rocksdb/db.h"
|
|
|
|
|
#include "rocksdb/env.h"
|
|
|
|
|
#include "rocksdb/memtablerep.h"
|
|
|
|
|
#include "rocksdb/transaction_log.h"
|
|
|
|
|
#include "table/scoped_arena_iterator.h"
|
|
|
|
|
#include "util/autovector.h"
|
EventLogger
Summary:
Here's my proposal for making our LOGs easier to read by machines.
The idea is to dump all events as JSON objects. JSON is easy to read by humans, but more importantly, it's easy to read by machines. That way, we can parse this, load into SQLite/mongo and then query or visualize.
I started with table_create and table_delete events, but if everybody agrees, I'll continue by adding more events (flush/compaction/etc etc)
Test Plan:
Ran db_bench. Observed:
2015/01/15-14:13:25.788019 1105ef000 EVENT_LOG_v1 {"time_micros": 1421360005788015, "event": "table_file_creation", "file_number": 12, "file_size": 1909699}
2015/01/15-14:13:25.956500 110740000 EVENT_LOG_v1 {"time_micros": 1421360005956498, "event": "table_file_deletion", "file_number": 12}
Reviewers: yhchiang, rven, dhruba, MarkCallaghan, lgalanis, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31647
10 years ago
|
|
|
#include "util/event_logger.h"
|
|
|
|
|
#include "util/stop_watch.h"
|
|
|
|
|
#include "util/thread_local.h"
|
|
|
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
class DBImpl;
|
|
|
|
|
class MemTable;
|
|
|
|
|
class SnapshotChecker;
|
|
|
|
|
class TableCache;
|
|
|
|
|
class Version;
|
|
|
|
|
class VersionEdit;
|
|
|
|
|
class VersionSet;
|
|
|
|
|
class Arena;
|
|
|
|
|
|
|
|
|
|
class FlushJob {
|
|
|
|
|
public:
|
|
|
|
|
// TODO(icanadi) make effort to reduce number of parameters here
|
|
|
|
|
// IMPORTANT: mutable_cf_options needs to be alive while FlushJob is alive
|
|
|
|
|
FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
|
|
|
|
|
const ImmutableDBOptions& db_options,
|
|
|
|
|
const MutableCFOptions& mutable_cf_options,
|
|
|
|
|
const EnvOptions env_options, VersionSet* versions,
|
|
|
|
|
InstrumentedMutex* db_mutex, std::atomic<bool>* shutting_down,
|
|
|
|
|
std::vector<SequenceNumber> existing_snapshots,
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
9 years ago
|
|
|
SequenceNumber earliest_write_conflict_snapshot,
|
|
|
|
|
SnapshotChecker* snapshot_checker, JobContext* job_context,
|
|
|
|
|
LogBuffer* log_buffer, Directory* db_directory,
|
|
|
|
|
Directory* output_file_directory, CompressionType output_compression,
|
|
|
|
|
Statistics* stats, EventLogger* event_logger, bool measure_io_stats);
|
|
|
|
|
|
|
|
|
|
~FlushJob();
|
|
|
|
|
|
|
|
|
|
// Require db_mutex held.
|
|
|
|
|
// Once PickMemTable() is called, either Run() or Cancel() has to be called.
|
|
|
|
|
void PickMemTable();
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
Status Run(LogsWithPrepTracker* prep_tracker = nullptr,
|
|
|
|
|
FileMetaData* file_meta = nullptr);
|
|
|
|
|
void Cancel();
|
|
|
|
|
TableProperties GetTableProperties() const { return table_properties_; }
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
void ReportStartedFlush();
|
|
|
|
|
void ReportFlushInputSize(const autovector<MemTable*>& mems);
|
|
|
|
|
void RecordFlushIOStats();
|
|
|
|
|
Status WriteLevel0Table();
|
|
|
|
|
const std::string& dbname_;
|
|
|
|
|
ColumnFamilyData* cfd_;
|
|
|
|
|
const ImmutableDBOptions& db_options_;
|
|
|
|
|
const MutableCFOptions& mutable_cf_options_;
|
|
|
|
|
const EnvOptions env_options_;
|
|
|
|
|
VersionSet* versions_;
|
|
|
|
|
InstrumentedMutex* db_mutex_;
|
|
|
|
|
std::atomic<bool>* shutting_down_;
|
|
|
|
|
std::vector<SequenceNumber> existing_snapshots_;
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
9 years ago
|
|
|
SequenceNumber earliest_write_conflict_snapshot_;
|
|
|
|
|
SnapshotChecker* snapshot_checker_;
|
|
|
|
|
JobContext* job_context_;
|
|
|
|
|
LogBuffer* log_buffer_;
|
|
|
|
|
Directory* db_directory_;
|
|
|
|
|
Directory* output_file_directory_;
|
|
|
|
|
CompressionType output_compression_;
|
|
|
|
|
Statistics* stats_;
|
EventLogger
Summary:
Here's my proposal for making our LOGs easier to read by machines.
The idea is to dump all events as JSON objects. JSON is easy to read by humans, but more importantly, it's easy to read by machines. That way, we can parse this, load into SQLite/mongo and then query or visualize.
I started with table_create and table_delete events, but if everybody agrees, I'll continue by adding more events (flush/compaction/etc etc)
Test Plan:
Ran db_bench. Observed:
2015/01/15-14:13:25.788019 1105ef000 EVENT_LOG_v1 {"time_micros": 1421360005788015, "event": "table_file_creation", "file_number": 12, "file_size": 1909699}
2015/01/15-14:13:25.956500 110740000 EVENT_LOG_v1 {"time_micros": 1421360005956498, "event": "table_file_deletion", "file_number": 12}
Reviewers: yhchiang, rven, dhruba, MarkCallaghan, lgalanis, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31647
10 years ago
|
|
|
EventLogger* event_logger_;
|
|
|
|
|
TableProperties table_properties_;
|
|
|
|
|
bool measure_io_stats_;
|
|
|
|
|
|
|
|
|
|
// Variables below are set by PickMemTable():
|
|
|
|
|
FileMetaData meta_;
|
|
|
|
|
autovector<MemTable*> mems_;
|
|
|
|
|
VersionEdit* edit_;
|
|
|
|
|
Version* base_;
|
|
|
|
|
bool pick_memtable_called;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
} // namespace rocksdb
|
