You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

177 lines
7.0 KiB

// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "rocksdb/utilities/option_change_migration.h"
#include "rocksdb/db.h"
namespace ROCKSDB_NAMESPACE {
namespace {
// Return a version of Options `opts` that allow us to open/write into a DB
// without triggering an automatic compaction or stalling. This is guaranteed
// by disabling automatic compactions and using huge values for stalling
// triggers.
Options GetNoCompactionOptions(const Options& opts) {
Options ret_opts = opts;
ret_opts.disable_auto_compactions = true;
ret_opts.level0_slowdown_writes_trigger = 999999;
ret_opts.level0_stop_writes_trigger = 999999;
ret_opts.soft_pending_compaction_bytes_limit = 0;
ret_opts.hard_pending_compaction_bytes_limit = 0;
return ret_opts;
}
Status OpenDb(const Options& options, const std::string& dbname,
std::unique_ptr<DB>* db) {
db->reset();
DB* tmpdb;
Status s = DB::Open(options, dbname, &tmpdb);
if (s.ok()) {
db->reset(tmpdb);
}
return s;
}
// l0_file_size specifies size of file on L0. Files will be range partitioned
// after a full compaction so they are likely qualified to put on L0. If
// left as 0, the files are compacted in a single file and put to L0. Otherwise,
// will try to compact the files as size l0_file_size.
Status CompactToLevel(const Options& options, const std::string& dbname,
int dest_level, uint64_t l0_file_size, bool need_reopen) {
std::unique_ptr<DB> db;
Options no_compact_opts = GetNoCompactionOptions(options);
if (dest_level == 0) {
if (l0_file_size == 0) {
// Single file.
l0_file_size = 999999999999999;
}
// L0 has strict sequenceID requirements to files to it. It's safer
// to only put one compacted file to there.
// This is only used for converting to universal compaction with
// only one level. In this case, compacting to one file is also
// optimal.
no_compact_opts.target_file_size_base = l0_file_size;
no_compact_opts.max_compaction_bytes = l0_file_size;
}
Status s = OpenDb(no_compact_opts, dbname, &db);
if (!s.ok()) {
return s;
}
CompactRangeOptions cro;
cro.change_level = true;
cro.target_level = dest_level;
if (dest_level == 0) {
// cannot use kForceOptimized because the compaction is expected to
// generate one output file
cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
}
s = db->CompactRange(cro, nullptr, nullptr);
if (s.ok() && need_reopen) {
// Need to restart DB to rewrite the manifest file.
// In order to open a DB with specific num_levels, the manifest file should
// contain no record that mentiones any level beyond num_levels. Issuing a
// full compaction will move all the data to a level not exceeding
// num_levels, but the manifest may still contain previous record mentioning
// a higher level. Reopening the DB will force the manifest to be rewritten
// so that those records will be cleared.
db.reset();
s = OpenDb(no_compact_opts, dbname, &db);
}
return s;
}
Status MigrateToUniversal(std::string dbname, const Options& old_opts,
const Options& new_opts) {
if (old_opts.num_levels <= new_opts.num_levels ||
old_opts.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
return Status::OK();
} else {
bool need_compact = false;
{
std::unique_ptr<DB> db;
Options opts = GetNoCompactionOptions(old_opts);
Status s = OpenDb(opts, dbname, &db);
if (!s.ok()) {
return s;
}
ColumnFamilyMetaData metadata;
db->GetColumnFamilyMetaData(&metadata);
if (!metadata.levels.empty() &&
metadata.levels.back().level >= new_opts.num_levels) {
need_compact = true;
}
}
if (need_compact) {
return CompactToLevel(old_opts, dbname, new_opts.num_levels - 1,
/*l0_file_size=*/0, true);
}
return Status::OK();
}
}
Status MigrateToLevelBase(std::string dbname, const Options& old_opts,
const Options& new_opts) {
if (!new_opts.level_compaction_dynamic_level_bytes) {
if (old_opts.num_levels == 1) {
return Status::OK();
}
// Compact everything to level 1 to guarantee it can be safely opened.
Options opts = old_opts;
opts.target_file_size_base = new_opts.target_file_size_base;
// Although sometimes we can open the DB with the new option without error,
// We still want to compact the files to avoid the LSM tree to stuck
// in bad shape. For example, if the user changed the level size
// multiplier from 4 to 8, with the same data, we will have fewer
// levels. Unless we issue a full comaction, the LSM tree may stuck
// with more levels than needed and it won't recover automatically.
return CompactToLevel(opts, dbname, 1, /*l0_file_size=*/0, true);
} else {
// Compact everything to the last level to guarantee it can be safely
// opened.
if (old_opts.num_levels == 1) {
return Status::OK();
} else if (new_opts.num_levels > old_opts.num_levels) {
// Dynamic level mode requires data to be put in the last level first.
return CompactToLevel(new_opts, dbname, new_opts.num_levels - 1,
/*l0_file_size=*/0, false);
} else {
Options opts = old_opts;
opts.target_file_size_base = new_opts.target_file_size_base;
return CompactToLevel(opts, dbname, new_opts.num_levels - 1,
/*l0_file_size=*/0, true);
}
}
}
} // namespace
Status OptionChangeMigration(std::string dbname, const Options& old_opts,
const Options& new_opts) {
if (old_opts.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
// LSM generated by FIFO compaction can be opened by any compaction.
return Status::OK();
} else if (new_opts.compaction_style ==
CompactionStyle::kCompactionStyleUniversal) {
return MigrateToUniversal(dbname, old_opts, new_opts);
} else if (new_opts.compaction_style ==
CompactionStyle::kCompactionStyleLevel) {
return MigrateToLevelBase(dbname, old_opts, new_opts);
} else if (new_opts.compaction_style ==
CompactionStyle::kCompactionStyleFIFO) {
uint64_t l0_file_size = 0;
if (new_opts.compaction_options_fifo.max_table_files_size > 0) {
// Create at least 8 files when max_table_files_size hits, so that the DB
// doesn't just disappear. This in fact violates the FIFO condition, but
// otherwise, the migrated DB is unlikley to be usable.
l0_file_size = new_opts.compaction_options_fifo.max_table_files_size / 8;
}
return CompactToLevel(old_opts, dbname, 0, l0_file_size, true);
} else {
return Status::NotSupported(
"Do not how to migrate to this compaction style");
}
}
} // namespace ROCKSDB_NAMESPACE