[RocksDB] cleanup EnvOptions

Summary:
This diff simplifies EnvOptions by treating it as POD, similar to Options.
- virtual functions are removed and member fields are accessed directly.
- StorageOptions is removed.
- Options.allow_readahead and Options.allow_readahead_compactions are deprecated.
- Unused global variables are removed: useOsBuffer, useFsReadAhead, useMmapRead, useMmapWrite

Test Plan: make check; db_stress

Reviewers: dhruba

CC: leveldb

Differential Revision: https://reviews.facebook.net/D11175
main
Haobo Xu 12 years ago
parent 5679107b07
commit bdf1085944
  1. 2
      db/builder.cc
  2. 4
      db/builder.h
  3. 12
      db/db_bench.cc
  4. 8
      db/db_impl.cc
  5. 2
      db/db_impl.h
  6. 7
      db/db_test.cc
  7. 2
      db/repair.cc
  8. 2
      db/table_cache.cc
  9. 5
      db/table_cache.h
  10. 2
      db/transaction_log_iterator_impl.cc
  11. 5
      db/transaction_log_iterator_impl.h
  12. 6
      db/version_set.cc
  13. 10
      db/version_set.h
  14. 3
      helpers/memenv/memenv_test.cc
  15. 47
      include/leveldb/env.h
  16. 2
      include/leveldb/options.h
  17. 2
      table/table_test.cc
  18. 13
      tools/db_stress.cc
  19. 2
      tools/sst_dump.cc
  20. 27
      util/env.cc
  21. 33
      util/env_posix.cc
  22. 7
      util/env_test.cc
  23. 8
      util/ldb_cmd.cc
  24. 72
      util/storage_options.h

@ -19,7 +19,7 @@ namespace leveldb {
Status BuildTable(const std::string& dbname, Status BuildTable(const std::string& dbname,
Env* env, Env* env,
const Options& options, const Options& options,
const StorageOptions& soptions, const EnvOptions& soptions,
TableCache* table_cache, TableCache* table_cache,
Iterator* iter, Iterator* iter,
FileMetaData* meta, FileMetaData* meta,

@ -8,7 +8,6 @@
#include "leveldb/comparator.h" #include "leveldb/comparator.h"
#include "leveldb/status.h" #include "leveldb/status.h"
#include "leveldb/types.h" #include "leveldb/types.h"
#include "util/storage_options.h"
namespace leveldb { namespace leveldb {
@ -16,6 +15,7 @@ struct Options;
struct FileMetaData; struct FileMetaData;
class Env; class Env;
class EnvOptions;
class Iterator; class Iterator;
class TableCache; class TableCache;
class VersionEdit; class VersionEdit;
@ -28,7 +28,7 @@ class VersionEdit;
extern Status BuildTable(const std::string& dbname, extern Status BuildTable(const std::string& dbname,
Env* env, Env* env,
const Options& options, const Options& options,
const StorageOptions& soptions, const EnvOptions& soptions,
TableCache* table_cache, TableCache* table_cache,
Iterator* iter, Iterator* iter,
FileMetaData* meta, FileMetaData* meta,

@ -1157,7 +1157,6 @@ unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "")
options.allow_readahead = FLAGS_use_fsreadahead; options.allow_readahead = FLAGS_use_fsreadahead;
options.allow_mmap_reads = FLAGS_use_mmap_reads; options.allow_mmap_reads = FLAGS_use_mmap_reads;
options.allow_mmap_writes = FLAGS_use_mmap_writes; options.allow_mmap_writes = FLAGS_use_mmap_writes;
options.allow_readahead_compactions = FLAGS_use_readahead_compactions;
options.advise_random_on_open = FLAGS_advise_random_on_open; options.advise_random_on_open = FLAGS_advise_random_on_open;
options.access_hint_on_compaction_start = FLAGS_compaction_fadvice; options.access_hint_on_compaction_start = FLAGS_compaction_fadvice;
@ -1714,7 +1713,7 @@ unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "")
void HeapProfile() { void HeapProfile() {
char fname[100]; char fname[100];
StorageOptions soptions; EnvOptions soptions;
snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_); snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_);
unique_ptr<WritableFile> file; unique_ptr<WritableFile> file;
Status s = FLAGS_env->NewWritableFile(fname, &file, soptions); Status s = FLAGS_env->NewWritableFile(fname, &file, soptions);
@ -1742,12 +1741,9 @@ int main(int argc, char** argv) {
leveldb::Options().max_background_compactions; leveldb::Options().max_background_compactions;
// Compression test code above refers to FLAGS_block_size // Compression test code above refers to FLAGS_block_size
FLAGS_block_size = leveldb::Options().block_size; FLAGS_block_size = leveldb::Options().block_size;
FLAGS_use_os_buffer = leveldb::StorageOptions().UseOsBuffer(); FLAGS_use_os_buffer = leveldb::EnvOptions().use_os_buffer;
FLAGS_use_fsreadahead = leveldb::StorageOptions().UseReadahead(); FLAGS_use_mmap_reads = leveldb::EnvOptions().use_mmap_reads;
FLAGS_use_mmap_reads = leveldb::StorageOptions().UseMmapReads(); FLAGS_use_mmap_writes = leveldb::EnvOptions().use_mmap_writes;
FLAGS_use_mmap_writes = leveldb::StorageOptions().UseMmapWrites();
FLAGS_use_readahead_compactions =
leveldb::StorageOptions().UseReadaheadCompactions();
std::string default_db_path; std::string default_db_path;

@ -2393,8 +2393,8 @@ Status DBImpl::MakeRoomForWrite(bool force) {
assert(versions_->PrevLogNumber() == 0); assert(versions_->PrevLogNumber() == 0);
uint64_t new_log_number = versions_->NewFileNumber(); uint64_t new_log_number = versions_->NewFileNumber();
unique_ptr<WritableFile> lfile; unique_ptr<WritableFile> lfile;
StorageOptions soptions(storage_options_); EnvOptions soptions(storage_options_);
soptions.DisableMmapWrites(); soptions.use_mmap_writes = false;
s = env_->NewWritableFile( s = env_->NewWritableFile(
LogFileName(dbname_, new_log_number), LogFileName(dbname_, new_log_number),
&lfile, &lfile,
@ -2668,7 +2668,7 @@ DB::~DB() { }
Status DB::Open(const Options& options, const std::string& dbname, Status DB::Open(const Options& options, const std::string& dbname,
DB** dbptr) { DB** dbptr) {
*dbptr = nullptr; *dbptr = nullptr;
StorageOptions soptions; EnvOptions soptions;
if (options.block_cache != nullptr && options.no_block_cache) { if (options.block_cache != nullptr && options.no_block_cache) {
return Status::InvalidArgument( return Status::InvalidArgument(
@ -2686,7 +2686,7 @@ Status DB::Open(const Options& options, const std::string& dbname,
if (s.ok()) { if (s.ok()) {
uint64_t new_log_number = impl->versions_->NewFileNumber(); uint64_t new_log_number = impl->versions_->NewFileNumber();
unique_ptr<WritableFile> lfile; unique_ptr<WritableFile> lfile;
soptions.DisableMmapWrites(); soptions.use_mmap_writes = false;
s = options.env->NewWritableFile(LogFileName(dbname, new_log_number), s = options.env->NewWritableFile(LogFileName(dbname, new_log_number),
&lfile, soptions); &lfile, soptions);
if (s.ok()) { if (s.ok()) {

@ -366,7 +366,7 @@ class DBImpl : public DB {
SequenceNumber last_flushed_sequence_; SequenceNumber last_flushed_sequence_;
// The options to access storage files // The options to access storage files
const StorageOptions storage_options_; const EnvOptions storage_options_;
// No copying allowed // No copying allowed
DBImpl(const DBImpl&); DBImpl(const DBImpl&);

@ -20,7 +20,6 @@
#include "util/mutexlock.h" #include "util/mutexlock.h"
#include "util/testharness.h" #include "util/testharness.h"
#include "util/testutil.h" #include "util/testutil.h"
#include "util/storage_options.h"
#include "utilities/merge_operators.h" #include "utilities/merge_operators.h"
namespace leveldb { namespace leveldb {
@ -606,7 +605,7 @@ TEST(DBTest, LevelLimitReopen) {
TEST(DBTest, Preallocation) { TEST(DBTest, Preallocation) {
const std::string src = dbname_ + "/alloc_test"; const std::string src = dbname_ + "/alloc_test";
unique_ptr<WritableFile> srcfile; unique_ptr<WritableFile> srcfile;
const StorageOptions soptions; const EnvOptions soptions;
ASSERT_OK(env_->NewWritableFile(src, &srcfile, soptions)); ASSERT_OK(env_->NewWritableFile(src, &srcfile, soptions));
srcfile->SetPreallocationBlockSize(1024 * 1024); srcfile->SetPreallocationBlockSize(1024 * 1024);
@ -2383,7 +2382,7 @@ TEST(DBTest, BloomFilter) {
TEST(DBTest, SnapshotFiles) { TEST(DBTest, SnapshotFiles) {
Options options = CurrentOptions(); Options options = CurrentOptions();
const StorageOptions soptions; const EnvOptions soptions;
options.write_buffer_size = 100000000; // Large write buffer options.write_buffer_size = 100000000; // Large write buffer
Reopen(&options); Reopen(&options);
@ -3299,7 +3298,7 @@ void BM_LogAndApply(int iters, int num_base_files) {
InternalKeyComparator cmp(BytewiseComparator()); InternalKeyComparator cmp(BytewiseComparator());
Options options; Options options;
StorageOptions sopt; EnvOptions sopt;
VersionSet vset(dbname, &options, sopt, nullptr, &cmp); VersionSet vset(dbname, &options, sopt, nullptr, &cmp);
ASSERT_OK(vset.Recover()); ASSERT_OK(vset.Recover());
VersionEdit vbase(vset.NumberLevels()); VersionEdit vbase(vset.NumberLevels());

@ -105,7 +105,7 @@ class Repairer {
std::vector<uint64_t> logs_; std::vector<uint64_t> logs_;
std::vector<TableInfo> tables_; std::vector<TableInfo> tables_;
uint64_t next_file_number_; uint64_t next_file_number_;
const StorageOptions storage_options_; const EnvOptions storage_options_;
Status FindFiles() { Status FindFiles() {
std::vector<std::string> filenames; std::vector<std::string> filenames;

@ -26,7 +26,7 @@ static void UnrefEntry(void* arg1, void* arg2) {
TableCache::TableCache(const std::string& dbname, TableCache::TableCache(const std::string& dbname,
const Options* options, const Options* options,
const StorageOptions& storage_options, const EnvOptions& storage_options,
int entries) int entries)
: env_(options->env), : env_(options->env),
dbname_(dbname), dbname_(dbname),

@ -14,7 +14,6 @@
#include "leveldb/cache.h" #include "leveldb/cache.h"
#include "port/port.h" #include "port/port.h"
#include "table/table.h" #include "table/table.h"
#include "util/storage_options.h"
namespace leveldb { namespace leveldb {
@ -23,7 +22,7 @@ class Env;
class TableCache { class TableCache {
public: public:
TableCache(const std::string& dbname, const Options* options, TableCache(const std::string& dbname, const Options* options,
const StorageOptions& storage_options, int entries); const EnvOptions& storage_options, int entries);
~TableCache(); ~TableCache();
// Return an iterator for the specified file number (the corresponding // Return an iterator for the specified file number (the corresponding
@ -58,7 +57,7 @@ class TableCache {
Env* const env_; Env* const env_;
const std::string dbname_; const std::string dbname_;
const Options* options_; const Options* options_;
const StorageOptions& storage_options_; const EnvOptions& storage_options_;
std::shared_ptr<Cache> cache_; std::shared_ptr<Cache> cache_;
Status FindTable(const EnvOptions& toptions, Status FindTable(const EnvOptions& toptions,

@ -7,7 +7,7 @@ namespace leveldb {
TransactionLogIteratorImpl::TransactionLogIteratorImpl( TransactionLogIteratorImpl::TransactionLogIteratorImpl(
const std::string& dbname, const std::string& dbname,
const Options* options, const Options* options,
const StorageOptions& soptions, const EnvOptions& soptions,
SequenceNumber& seq, SequenceNumber& seq,
std::unique_ptr<std::vector<LogFile>> files, std::unique_ptr<std::vector<LogFile>> files,
SequenceNumber const * const lastFlushedSequence) : SequenceNumber const * const lastFlushedSequence) :

@ -10,7 +10,6 @@
#include "leveldb/transaction_log_iterator.h" #include "leveldb/transaction_log_iterator.h"
#include "db/log_file.h" #include "db/log_file.h"
#include "db/log_reader.h" #include "db/log_reader.h"
#include "util/storage_options.h"
namespace leveldb { namespace leveldb {
@ -26,7 +25,7 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
public: public:
TransactionLogIteratorImpl(const std::string& dbname, TransactionLogIteratorImpl(const std::string& dbname,
const Options* options, const Options* options,
const StorageOptions& soptions, const EnvOptions& soptions,
SequenceNumber& seqNum, SequenceNumber& seqNum,
std::unique_ptr<std::vector<LogFile>> files, std::unique_ptr<std::vector<LogFile>> files,
SequenceNumber const * const lastFlushedSequence); SequenceNumber const * const lastFlushedSequence);
@ -42,7 +41,7 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
private: private:
const std::string& dbname_; const std::string& dbname_;
const Options* options_; const Options* options_;
const StorageOptions& soptions_; const EnvOptions& soptions_;
const uint64_t startingSequenceNumber_; const uint64_t startingSequenceNumber_;
std::unique_ptr<std::vector<LogFile>> files_; std::unique_ptr<std::vector<LogFile>> files_;
bool started_; bool started_;

@ -198,7 +198,7 @@ static Iterator* GetFileIterator(void* arg,
} }
Iterator* Version::NewConcatenatingIterator(const ReadOptions& options, Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
const StorageOptions& soptions, const EnvOptions& soptions,
int level) const { int level) const {
return NewTwoLevelIterator( return NewTwoLevelIterator(
new LevelFileNumIterator(vset_->icmp_, &files_[level]), new LevelFileNumIterator(vset_->icmp_, &files_[level]),
@ -206,7 +206,7 @@ Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
} }
void Version::AddIterators(const ReadOptions& options, void Version::AddIterators(const ReadOptions& options,
const StorageOptions& soptions, const EnvOptions& soptions,
std::vector<Iterator*>* iters) { std::vector<Iterator*>* iters) {
// Merge all level zero files together since they may overlap // Merge all level zero files together since they may overlap
for (size_t i = 0; i < files_[0].size(); i++) { for (size_t i = 0; i < files_[0].size(); i++) {
@ -971,7 +971,7 @@ class VersionSet::Builder {
VersionSet::VersionSet(const std::string& dbname, VersionSet::VersionSet(const std::string& dbname,
const Options* options, const Options* options,
const StorageOptions& storage_options, const EnvOptions& storage_options,
TableCache* table_cache, TableCache* table_cache,
const InternalKeyComparator* cmp) const InternalKeyComparator* cmp)
: env_(options->env), : env_(options->env),

@ -63,7 +63,7 @@ class Version {
// Append to *iters a sequence of iterators that will // Append to *iters a sequence of iterators that will
// yield the contents of this Version when merged together. // yield the contents of this Version when merged together.
// REQUIRES: This version has been saved (see VersionSet::SaveTo) // REQUIRES: This version has been saved (see VersionSet::SaveTo)
void AddIterators(const ReadOptions&, const StorageOptions& soptions, void AddIterators(const ReadOptions&, const EnvOptions& soptions,
std::vector<Iterator*>* iters); std::vector<Iterator*>* iters);
// Lookup the value for key. If found, store it in *val and // Lookup the value for key. If found, store it in *val and
@ -138,7 +138,7 @@ class Version {
class LevelFileNumIterator; class LevelFileNumIterator;
Iterator* NewConcatenatingIterator(const ReadOptions&, Iterator* NewConcatenatingIterator(const ReadOptions&,
const StorageOptions& soptions, const EnvOptions& soptions,
int level) const; int level) const;
VersionSet* vset_; // VersionSet to which this Version belongs VersionSet* vset_; // VersionSet to which this Version belongs
@ -207,7 +207,7 @@ class VersionSet {
public: public:
VersionSet(const std::string& dbname, VersionSet(const std::string& dbname,
const Options* options, const Options* options,
const StorageOptions& storage_options, const EnvOptions& storage_options,
TableCache* table_cache, TableCache* table_cache,
const InternalKeyComparator*); const InternalKeyComparator*);
~VersionSet(); ~VersionSet();
@ -458,11 +458,11 @@ class VersionSet {
uint64_t last_observed_manifest_size_; uint64_t last_observed_manifest_size_;
// storage options for all reads and writes except compactions // storage options for all reads and writes except compactions
const StorageOptions& storage_options_; const EnvOptions& storage_options_;
// storage options used for compactions. This is a copy of // storage options used for compactions. This is a copy of
// storage_options_ but with readaheads set to readahead_compactions_. // storage_options_ but with readaheads set to readahead_compactions_.
const StorageOptions storage_options_compactions_; const EnvOptions storage_options_compactions_;
// No copying allowed // No copying allowed
VersionSet(const VersionSet&); VersionSet(const VersionSet&);

@ -8,7 +8,6 @@
#include "leveldb/db.h" #include "leveldb/db.h"
#include "leveldb/env.h" #include "leveldb/env.h"
#include "util/testharness.h" #include "util/testharness.h"
#include "util/storage_options.h"
#include <memory> #include <memory>
#include <string> #include <string>
#include <vector> #include <vector>
@ -18,7 +17,7 @@ namespace leveldb {
class MemEnvTest { class MemEnvTest {
public: public:
Env* env_; Env* env_;
const StorageOptions soptions_; const EnvOptions soptions_;
MemEnvTest() MemEnvTest()
: env_(NewMemEnv(Env::Default())) { : env_(NewMemEnv(Env::Default())) {

@ -23,16 +23,40 @@
namespace leveldb { namespace leveldb {
class FileLock; class FileLock;
class EnvOptions;
class Logger; class Logger;
class RandomAccessFile; class RandomAccessFile;
class SequentialFile; class SequentialFile;
class Slice; class Slice;
class WritableFile; class WritableFile;
class Options;
using std::unique_ptr; using std::unique_ptr;
using std::shared_ptr; using std::shared_ptr;
// Options while opening a file to read/write
struct EnvOptions {
// construct with default Options
EnvOptions();
// construct from Options
EnvOptions(const Options& options);
// If true, then allow caching of data in environment buffers
bool use_os_buffer;
// If true, then use mmap to read data
bool use_mmap_reads;
// If true, then use mmap to write data
bool use_mmap_writes;
// If true, set the FD_CLOEXEC on open fd.
bool set_fd_cloexec;
};
class Env { class Env {
public: public:
Env() { } Env() { }
@ -375,27 +399,6 @@ class FileLock {
void operator=(const FileLock&); void operator=(const FileLock&);
}; };
// Options while opening a file to read/write
class EnvOptions {
public:
virtual ~EnvOptions() {}
// If true, then allow caching of data in environment buffers
virtual bool UseOsBuffer() const = 0;
// If true, then allow the environment to readahead data
virtual bool UseReadahead() const = 0;
// If true, then use mmap to read data
virtual bool UseMmapReads() const = 0;
// If true, then use mmap to write data
virtual bool UseMmapWrites() const = 0;
// If true, set the FD_CLOEXEC on open fd.
virtual bool IsFDCloseOnExec() const = 0;
};
// Log the specified data to *info_log if info_log is non-nullptr. // Log the specified data to *info_log if info_log is non-nullptr.
extern void Log(const shared_ptr<Logger>& info_log, const char* format, ...) extern void Log(const shared_ptr<Logger>& info_log, const char* format, ...)
# if defined(__GNUC__) || defined(__clang__) # if defined(__GNUC__) || defined(__clang__)

@ -409,11 +409,13 @@ struct Options {
// Reading a single block from a file can cause the OS/FS to start // Reading a single block from a file can cause the OS/FS to start
// readaheads of other blocks from the file. Default: true // readaheads of other blocks from the file. Default: true
// Note: Deprecated
bool allow_readahead; bool allow_readahead;
// The reads triggered by compaction allows data to be readahead // The reads triggered by compaction allows data to be readahead
// by the OS/FS. This overrides the setting of 'allow_readahead' // by the OS/FS. This overrides the setting of 'allow_readahead'
// for compaction-reads. Default: true // for compaction-reads. Default: true
// Note: Deprecated
bool allow_readahead_compactions; bool allow_readahead_compactions;
// Allow the OS to mmap file for reading. Default: false // Allow the OS to mmap file for reading. Default: false

@ -293,7 +293,7 @@ class TableConstructor: public Constructor {
TableConstructor(); TableConstructor();
static uint64_t cur_uniq_id_; static uint64_t cur_uniq_id_;
const StorageOptions soptions; const EnvOptions soptions;
}; };
uint64_t TableConstructor::cur_uniq_id_ = 1; uint64_t TableConstructor::cur_uniq_id_ = 1;

@ -177,10 +177,6 @@ static uint32_t FLAGS_log2_keys_per_lock = 2; // implies 2^2 keys per lock
// Percentage of times we want to purge redundant keys in memory before flushing // Percentage of times we want to purge redundant keys in memory before flushing
static uint32_t FLAGS_purge_redundant_percent = 50; static uint32_t FLAGS_purge_redundant_percent = 50;
extern bool useOsBuffer;
extern bool useFsReadAhead;
extern bool useMmapRead;
namespace leveldb { namespace leveldb {
class StressTest; class StressTest;
@ -1069,15 +1065,6 @@ int main(int argc, char** argv) {
} else if (sscanf(argv[i], "--verify_checksum=%d%c", &n, &junk) == 1 && } else if (sscanf(argv[i], "--verify_checksum=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) { (n == 0 || n == 1)) {
FLAGS_verify_checksum = n; FLAGS_verify_checksum = n;
} else if (sscanf(argv[i], "--bufferedio=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) {
useOsBuffer = n;
} else if (sscanf(argv[i], "--mmap_read=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) {
useMmapRead = n;
} else if (sscanf(argv[i], "--readhead=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) {
useFsReadAhead = n;
} else if (sscanf(argv[i], "--statistics=%d%c", &n, &junk) == 1 && } else if (sscanf(argv[i], "--statistics=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) { (n == 0 || n == 1)) {
if (n == 1) { if (n == 1) {

@ -34,7 +34,7 @@ private:
uint64_t read_num_; uint64_t read_num_;
bool verify_checksum_; bool verify_checksum_;
bool output_hex_; bool output_hex_;
StorageOptions soptions_; EnvOptions soptions_;
}; };
SstFileReader::SstFileReader(std::string file_path, SstFileReader::SstFileReader(std::string file_path,

@ -3,7 +3,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "leveldb/env.h" #include "leveldb/env.h"
#include "util/storage_options.h" #include "leveldb/options.h"
namespace leveldb { namespace leveldb {
@ -47,7 +47,7 @@ static Status DoWriteStringToFile(Env* env, const Slice& data,
const std::string& fname, const std::string& fname,
bool should_sync) { bool should_sync) {
unique_ptr<WritableFile> file; unique_ptr<WritableFile> file;
StorageOptions soptions; EnvOptions soptions;
Status s = env->NewWritableFile(fname, &file, soptions); Status s = env->NewWritableFile(fname, &file, soptions);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
@ -73,7 +73,7 @@ Status WriteStringToFileSync(Env* env, const Slice& data,
} }
Status ReadFileToString(Env* env, const std::string& fname, std::string* data) { Status ReadFileToString(Env* env, const std::string& fname, std::string* data) {
StorageOptions soptions; EnvOptions soptions;
data->clear(); data->clear();
unique_ptr<SequentialFile> file; unique_ptr<SequentialFile> file;
Status s = env->NewSequentialFile(fname, &file, soptions); Status s = env->NewSequentialFile(fname, &file, soptions);
@ -100,4 +100,25 @@ Status ReadFileToString(Env* env, const std::string& fname, std::string* data) {
EnvWrapper::~EnvWrapper() { EnvWrapper::~EnvWrapper() {
} }
namespace { // anonymous namespace
void AssignEnvOptions(EnvOptions* env_options, const Options& options) {
env_options->use_os_buffer = options.allow_os_buffer;
env_options->use_mmap_reads = options.allow_mmap_reads;
env_options->use_mmap_writes = options.allow_mmap_writes;
env_options->set_fd_cloexec = options.is_fd_close_on_exec;
}
}
EnvOptions::EnvOptions(const Options& options) {
AssignEnvOptions(this, options);
}
EnvOptions::EnvOptions() {
Options options;
AssignEnvOptions(this, options);
}
} // namespace leveldb } // namespace leveldb

@ -45,11 +45,6 @@
#define EXT4_SUPER_MAGIC 0xEF53 #define EXT4_SUPER_MAGIC 0xEF53
#endif #endif
bool useOsBuffer = 1; // cache data in OS buffers
bool useFsReadAhead = 1; // allow filesystem to do readaheads
bool useMmapRead = 0; // do not use mmaps for reading files
bool useMmapWrite = 1; // use mmaps for appending to files
// This is only set from db_stress.cc and for testing only. // This is only set from db_stress.cc and for testing only.
// If non-zero, kill at various points in source code with probability 1/this // If non-zero, kill at various points in source code with probability 1/this
int leveldb_kill_odds = 0; int leveldb_kill_odds = 0;
@ -111,8 +106,8 @@ class PosixSequentialFile: public SequentialFile {
PosixSequentialFile(const std::string& fname, FILE* f, PosixSequentialFile(const std::string& fname, FILE* f,
const EnvOptions& options) const EnvOptions& options)
: filename_(fname), file_(f), fd_(fileno(f)), : filename_(fname), file_(f), fd_(fileno(f)),
use_os_buffer_(options.UseOsBuffer()) { use_os_buffer_(options.use_os_buffer) {
assert(!options.UseMmapReads()); assert(!options.use_mmap_reads);
} }
virtual ~PosixSequentialFile() { fclose(file_); } virtual ~PosixSequentialFile() { fclose(file_); }
@ -154,11 +149,8 @@ class PosixRandomAccessFile: public RandomAccessFile {
public: public:
PosixRandomAccessFile(const std::string& fname, int fd, PosixRandomAccessFile(const std::string& fname, int fd,
const EnvOptions& options) const EnvOptions& options)
: filename_(fname), fd_(fd), use_os_buffer_(options.UseOsBuffer()) { : filename_(fname), fd_(fd), use_os_buffer_(options.use_os_buffer) {
assert(!options.UseMmapReads()); assert(!options.use_mmap_reads);
if (!options.UseReadahead()) { // disable read-aheads
posix_fadvise(fd, 0, 0, POSIX_FADV_RANDOM);
}
} }
virtual ~PosixRandomAccessFile() { close(fd_); } virtual ~PosixRandomAccessFile() { close(fd_); }
@ -245,9 +237,8 @@ class PosixMmapReadableFile: public RandomAccessFile {
PosixMmapReadableFile(const std::string& fname, void* base, size_t length, PosixMmapReadableFile(const std::string& fname, void* base, size_t length,
const EnvOptions& options) const EnvOptions& options)
: filename_(fname), mmapped_region_(base), length_(length) { : filename_(fname), mmapped_region_(base), length_(length) {
assert(options.UseMmapReads()); assert(options.use_mmap_reads);
assert(options.UseOsBuffer()); assert(options.use_os_buffer);
assert(options.UseReadahead());
} }
virtual ~PosixMmapReadableFile() { munmap(mmapped_region_, length_); } virtual ~PosixMmapReadableFile() { munmap(mmapped_region_, length_); }
@ -359,7 +350,7 @@ class PosixMmapFile : public WritableFile {
file_offset_(0), file_offset_(0),
pending_sync_(false) { pending_sync_(false) {
assert((page_size & (page_size - 1)) == 0); assert((page_size & (page_size - 1)) == 0);
assert(options.UseMmapWrites()); assert(options.use_mmap_writes);
} }
@ -524,7 +515,7 @@ class PosixWritableFile : public WritableFile {
filesize_(0), filesize_(0),
pending_sync_(false), pending_sync_(false),
pending_fsync_(false) { pending_fsync_(false) {
assert(!options.UseMmapWrites()); assert(!options.use_mmap_writes);
} }
~PosixWritableFile() { ~PosixWritableFile() {
@ -703,7 +694,7 @@ class PosixEnv : public Env {
} }
void SetFD_CLOEXEC(int fd, const EnvOptions* options) { void SetFD_CLOEXEC(int fd, const EnvOptions* options) {
if ((options == nullptr || options->IsFDCloseOnExec()) && fd > 0) { if ((options == nullptr || options->set_fd_cloexec) && fd > 0) {
fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
} }
} }
@ -733,7 +724,7 @@ class PosixEnv : public Env {
SetFD_CLOEXEC(fd, &options); SetFD_CLOEXEC(fd, &options);
if (fd < 0) { if (fd < 0) {
s = IOError(fname, errno); s = IOError(fname, errno);
} else if (options.UseMmapReads() && sizeof(void*) >= 8) { } else if (options.use_mmap_reads && sizeof(void*) >= 8) {
// Use of mmap for random reads has been removed because it // Use of mmap for random reads has been removed because it
// kills performance when storage is fast. // kills performance when storage is fast.
// Use mmap when virtual address-space is plentiful. // Use mmap when virtual address-space is plentiful.
@ -764,7 +755,7 @@ class PosixEnv : public Env {
s = IOError(fname, errno); s = IOError(fname, errno);
} else { } else {
SetFD_CLOEXEC(fd, &options); SetFD_CLOEXEC(fd, &options);
if (options.UseMmapWrites()) { if (options.use_mmap_writes) {
if (!checkedDiskForMmap_) { if (!checkedDiskForMmap_) {
// this will be executed once in the program's lifetime. // this will be executed once in the program's lifetime.
// do not use mmapWrite on non ext-3/xfs/tmpfs systems. // do not use mmapWrite on non ext-3/xfs/tmpfs systems.
@ -774,7 +765,7 @@ class PosixEnv : public Env {
checkedDiskForMmap_ = true; checkedDiskForMmap_ = true;
} }
} }
if (options.UseMmapWrites() && !forceMmapOff) { if (options.use_mmap_writes && !forceMmapOff) {
result->reset(new PosixMmapFile(fname, fd, page_size_, options)); result->reset(new PosixMmapFile(fname, fd, page_size_, options));
} else { } else {
result->reset(new PosixWritableFile(fname, fd, 65536, options)); result->reset(new PosixWritableFile(fname, fd, 65536, options));

@ -8,7 +8,6 @@
#include "port/port.h" #include "port/port.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/testharness.h" #include "util/testharness.h"
#include "util/storage_options.h"
namespace leveldb { namespace leveldb {
@ -120,7 +119,7 @@ char temp_id[MAX_ID_SIZE];
TEST(EnvPosixTest, RandomAccessUniqueID) { TEST(EnvPosixTest, RandomAccessUniqueID) {
// Create file. // Create file.
const StorageOptions soptions; const EnvOptions soptions;
std::string fname = test::TmpDir() + "/" + "testfile"; std::string fname = test::TmpDir() + "/" + "testfile";
unique_ptr<WritableFile> wfile; unique_ptr<WritableFile> wfile;
ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions)); ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions));
@ -174,7 +173,7 @@ bool HasPrefix(const std::unordered_set<std::string>& ss) {
TEST(EnvPosixTest, RandomAccessUniqueIDConcurrent) { TEST(EnvPosixTest, RandomAccessUniqueIDConcurrent) {
// Check whether a bunch of concurrently existing files have unique IDs. // Check whether a bunch of concurrently existing files have unique IDs.
const StorageOptions soptions; const EnvOptions soptions;
// Create the files // Create the files
std::vector<std::string> fnames; std::vector<std::string> fnames;
@ -210,7 +209,7 @@ TEST(EnvPosixTest, RandomAccessUniqueIDConcurrent) {
} }
TEST(EnvPosixTest, RandomAccessUniqueIDDeletes) { TEST(EnvPosixTest, RandomAccessUniqueIDDeletes) {
const StorageOptions soptions; const EnvOptions soptions;
std::string fname = test::TmpDir() + "/" + "testfile"; std::string fname = test::TmpDir() + "/" + "testfile";
// Check that after file is deleted we don't get same ID again in a new file. // Check that after file is deleted we don't get same ID again in a new file.

@ -503,7 +503,7 @@ void ManifestDumpCommand::DoCommand() {
} }
Options options; Options options;
StorageOptions sopt; EnvOptions sopt;
std::string file(manifestfile); std::string file(manifestfile);
std::string dbname("dummy"); std::string dbname("dummy");
TableCache* tc = new TableCache(dbname, &options, sopt, 10); TableCache* tc = new TableCache(dbname, &options, sopt, 10);
@ -691,7 +691,7 @@ Options ReduceDBLevelsCommand::PrepareOptionsForOpenDB() {
Status ReduceDBLevelsCommand::GetOldNumOfLevels(Options& opt, Status ReduceDBLevelsCommand::GetOldNumOfLevels(Options& opt,
int* levels) { int* levels) {
StorageOptions soptions; EnvOptions soptions;
TableCache tc(db_path_, &opt, soptions, 10); TableCache tc(db_path_, &opt, soptions, 10);
const InternalKeyComparator cmp(opt.comparator); const InternalKeyComparator cmp(opt.comparator);
VersionSet versions(db_path_, &opt, soptions, &tc, &cmp); VersionSet versions(db_path_, &opt, soptions, &tc, &cmp);
@ -748,7 +748,7 @@ void ReduceDBLevelsCommand::DoCommand() {
db_->CompactRange(nullptr, nullptr); db_->CompactRange(nullptr, nullptr);
CloseDB(); CloseDB();
StorageOptions soptions; EnvOptions soptions;
TableCache tc(db_path_, &opt, soptions, 10); TableCache tc(db_path_, &opt, soptions, 10);
const InternalKeyComparator cmp(opt.comparator); const InternalKeyComparator cmp(opt.comparator);
VersionSet versions(db_path_, &opt, soptions, &tc, &cmp); VersionSet versions(db_path_, &opt, soptions, &tc, &cmp);
@ -840,7 +840,7 @@ void WALDumperCommand::DoCommand() {
unique_ptr<SequentialFile> file; unique_ptr<SequentialFile> file;
Env* env_ = Env::Default(); Env* env_ = Env::Default();
StorageOptions soptions; EnvOptions soptions;
Status status = env_->NewSequentialFile(wal_file_, &file, soptions); Status status = env_->NewSequentialFile(wal_file_, &file, soptions);
if (!status.ok()) { if (!status.ok()) {
exec_state_ = LDBCommandExecuteResult::FAILED("Failed to open WAL file " + exec_state_ = LDBCommandExecuteResult::FAILED("Failed to open WAL file " +

@ -1,72 +0,0 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
//
#ifndef STORAGE_LEVELDB_UTIL_STORAGE_OPTIONS_H_
#define STORAGE_LEVELDB_UTIL_STORAGE_OPTIONS_H_
#include <string>
#include <stdint.h>
#include "leveldb/env.h"
#include "leveldb/options.h"
namespace leveldb {
// Environment Options that are used to read files from storage
class StorageOptions : public EnvOptions {
public:
/* implicit */ StorageOptions(const Options& opt) :
data_in_os_(opt.allow_os_buffer),
fs_readahead_(opt.allow_readahead),
readahead_compactions_(opt.allow_readahead_compactions),
use_mmap_reads_(opt.allow_mmap_reads),
use_mmap_writes_(opt.allow_mmap_writes),
set_fd_cloexec_(opt.is_fd_close_on_exec) {
}
// copy constructor with readaheads set to readahead_compactions_
StorageOptions(const StorageOptions& opt) {
data_in_os_ = opt.UseOsBuffer();
fs_readahead_ = opt.UseReadaheadCompactions();
readahead_compactions_ = opt.UseReadaheadCompactions();
use_mmap_reads_ = opt.UseMmapReads();
use_mmap_writes_ = opt.UseMmapWrites();
set_fd_cloexec_ = opt.IsFDCloseOnExec();
}
// constructor with default options
StorageOptions() {
Options opt;
data_in_os_ = opt.allow_os_buffer;
fs_readahead_ = opt.allow_readahead;
readahead_compactions_ = fs_readahead_;
use_mmap_reads_ = opt.allow_mmap_reads;
use_mmap_writes_ = opt.allow_mmap_writes;
set_fd_cloexec_ = opt.is_fd_close_on_exec;
}
virtual ~StorageOptions() {}
bool UseOsBuffer() const { return data_in_os_; };
bool UseReadahead() const { return fs_readahead_; };
bool UseMmapReads() const { return use_mmap_reads_; }
bool UseMmapWrites() const { return use_mmap_writes_; }
bool UseReadaheadCompactions() const { return readahead_compactions_;}
bool IsFDCloseOnExec() const { return set_fd_cloexec_;}
void DisableMmapWrites() {
use_mmap_writes_ = false;
}
private:
bool data_in_os_;
bool fs_readahead_;
bool readahead_compactions_;
bool use_mmap_reads_;
bool use_mmap_writes_;
bool set_fd_cloexec_;
};
} // namespace leveldb
#endif // STORAGE_LEVELDB_UTIL_STORAGE_OPTIONS_H_
Loading…
Cancel
Save