From 5a9b4d74354c1499839728831fecafd9b75b0af5 Mon Sep 17 00:00:00 2001 From: Maysam Yabandeh Date: Thu, 1 Jun 2017 15:30:27 -0700 Subject: [PATCH] Retire memenv https://github.com/facebook/rocksdb/pull/2082 Summary: This is a manual commit of this PR: Retire InMemoryEnv in favor of MockEnv #2082 With MockEnv doing the same yet being more mature, InMemoryEnv is redundant. Reviewed By: IslamAbdelRahman Differential Revision: D5162323 fbshipit-source-id: 59fd0082a891dc99cc531e4da9d68bf891eae3f5 --- CMakeLists.txt | 3 +- TARGETS | 20 +-- env/memenv.cc | 451 ------------------------------------------------ env/mock_env.cc | 10 ++ src.mk | 3 +- 5 files changed, 20 insertions(+), 467 deletions(-) delete mode 100644 env/memenv.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 15784fa1a..20d9fdfab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -381,7 +381,7 @@ set(SOURCES env/env.cc env/env_chroot.cc env/env_hdfs.cc - env/memenv.cc + env/mock_env.cc memtable/hash_cuckoo_rep.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc @@ -745,7 +745,6 @@ endforeach(sourcefile ${BENCHMARKS}) # and linked to tests. Add test only code that is not #ifdefed for Release here. set(TESTUTIL_SOURCE db/db_test_util.cc - env/mock_env.cc monitoring/thread_status_updater_debug.cc table/mock_table.cc util/fault_injection_test_env.cc diff --git a/TARGETS b/TARGETS index 89056edd2..93c426e10 100644 --- a/TARGETS +++ b/TARGETS @@ -46,7 +46,7 @@ rocksdb_preprocessor_flags = [ cpp_library( name = "rocksdb_lib", - auto_headers = AutoHeaders.RECURSIVE_GLOB, + headers = AutoHeaders.RECURSIVE_GLOB, srcs = [ "cache/clock_cache.cc", "cache/lru_cache.cc", @@ -108,7 +108,7 @@ cpp_library( "env/env_hdfs.cc", "env/env_posix.cc", "env/io_posix.cc", - "env/memenv.cc", + "env/mock_env.cc", "memtable/hash_cuckoo_rep.cc", "memtable/hash_linklist_rep.cc", "memtable/hash_skiplist_rep.cc", @@ -201,7 +201,6 @@ cpp_library( "utilities/blob_db/blob_db.cc", "utilities/blob_db/blob_db_impl.cc", "utilities/blob_db/blob_db_options_impl.cc", - "utilities/blob_db/blob_dump_tool.cc", "utilities/blob_db/blob_file.cc", "utilities/blob_db/blob_log_reader.cc", "utilities/blob_db/blob_log_writer.cc", @@ -260,9 +259,8 @@ cpp_library( cpp_library( name = "rocksdb_test_lib", - auto_headers = AutoHeaders.RECURSIVE_GLOB, + headers = AutoHeaders.RECURSIVE_GLOB, srcs = [ - "env/mock_env.cc", "table/mock_table.cc", "util/fault_injection_test_env.cc", "util/testharness.cc", @@ -280,7 +278,7 @@ cpp_library( cpp_library( name = "rocksdb_tools_lib", - auto_headers = AutoHeaders.RECURSIVE_GLOB, + headers = AutoHeaders.RECURSIVE_GLOB, srcs = [ "tools/db_bench_tool.cc", "util/testutil.cc", @@ -293,7 +291,7 @@ cpp_library( cpp_library( name = "env_basic_test_lib", - auto_headers = AutoHeaders.RECURSIVE_GLOB, + headers = AutoHeaders.RECURSIVE_GLOB, srcs = ["env/env_basic_test.cc"], deps = [":rocksdb_test_lib"], preprocessor_flags = rocksdb_preprocessor_flags, @@ -334,6 +332,7 @@ ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'], ['object_registry_test', 'utilities/object_registry_test.cc', 'serial'], ['auto_roll_logger_test', 'util/auto_roll_logger_test.cc', 'serial'], ['dbformat_test', 'db/dbformat_test.cc', 'serial'], + ['db_write_test', 'db/db_write_test.cc', 'serial'], ['write_batch_with_index_test', 'utilities/write_batch_with_index/write_batch_with_index_test.cc', 'serial'], @@ -458,9 +457,8 @@ ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'], ['version_edit_test', 'db/version_edit_test.cc', 'serial'], ['skiplist_test', 'memtable/skiplist_test.cc', 'serial'], ['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'], - ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial'], - ['blob_db_test', 'utilities/blob_db/blob_db_test.cc', 'serial'], - ['db_write_test', 'db/db_write_test.cc', 'serial']] + ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial']] + # Generate a test rule for each entry in ROCKS_TESTS for test_cfg in ROCKS_TESTS: @@ -489,12 +487,10 @@ custom_unittest( name = "make_rocksdbjavastatic", type = "simple", command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"], - deps=[], ) custom_unittest( name = "make_rocksdb_lite_release", type = "simple", command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"], - deps=[], ) diff --git a/env/memenv.cc b/env/memenv.cc deleted file mode 100644 index 03dfe6cf9..000000000 --- a/env/memenv.cc +++ /dev/null @@ -1,451 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include - -#include -#include -#include -#include - -#include "rocksdb/env.h" -#include "rocksdb/status.h" -#include "port/port.h" -#include "util/mutexlock.h" - -namespace rocksdb { - -#ifndef ROCKSDB_LITE - -namespace { - -std::string NormalizeFileName(const std::string fname) { - if (fname.find("//") == std::string::npos) { - return fname; - } - std::string out_name = ""; - bool is_slash = false; - for (char c : fname) { - if (c == '/' && is_slash) { - continue; - } - out_name.append(1, c); - if (c == '/') { - is_slash = true; - } else { - is_slash = false; - } - } - return out_name; -} - -class FileState { - public: - // FileStates are reference counted. The initial reference count is zero - // and the caller must call Ref() at least once. - FileState() : refs_(0), size_(0) {} - - // Increase the reference count. - void Ref() { - MutexLock lock(&refs_mutex_); - ++refs_; - } - - // Decrease the reference count. Delete if this is the last reference. - void Unref() { - bool do_delete = false; - - { - MutexLock lock(&refs_mutex_); - --refs_; - assert(refs_ >= 0); - if (refs_ <= 0) { - do_delete = true; - } - } - - if (do_delete) { - delete this; - } - } - - uint64_t Size() const { return size_; } - - Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { - const uint64_t available = size_ - std::min(size_, offset); - if (n > available) { - n = available; - } - if (n == 0) { - *result = Slice(); - return Status::OK(); - } - - size_t block = offset / kBlockSize; - size_t block_offset = offset % kBlockSize; - - if (n <= kBlockSize - block_offset) { - // The requested bytes are all in the first block. - *result = Slice(blocks_[block] + block_offset, n); - return Status::OK(); - } - - size_t bytes_to_copy = n; - char* dst = scratch; - - while (bytes_to_copy > 0) { - size_t avail = kBlockSize - block_offset; - if (avail > bytes_to_copy) { - avail = bytes_to_copy; - } - memcpy(dst, blocks_[block] + block_offset, avail); - - bytes_to_copy -= avail; - dst += avail; - block++; - block_offset = 0; - } - - *result = Slice(scratch, n); - return Status::OK(); - } - - Status Append(const Slice& data) { - const char* src = data.data(); - size_t src_len = data.size(); - - while (src_len > 0) { - size_t avail; - size_t offset = size_ % kBlockSize; - - if (offset != 0) { - // There is some room in the last block. - avail = kBlockSize - offset; - } else { - // No room in the last block; push new one. - blocks_.push_back(new char[kBlockSize]); - avail = kBlockSize; - } - - if (avail > src_len) { - avail = src_len; - } - memcpy(blocks_.back() + offset, src, avail); - src_len -= avail; - src += avail; - size_ += avail; - } - - return Status::OK(); - } - - private: - // Private since only Unref() should be used to delete it. - ~FileState() { - for (std::vector::iterator i = blocks_.begin(); i != blocks_.end(); - ++i) { - delete [] *i; - } - } - - // No copying allowed. - FileState(const FileState&); - void operator=(const FileState&); - - port::Mutex refs_mutex_; - int refs_; // Protected by refs_mutex_; - - // The following fields are not protected by any mutex. They are only mutable - // while the file is being written, and concurrent access is not allowed - // to writable files. - std::vector blocks_; - uint64_t size_; - - enum { kBlockSize = 8 * 1024 }; -}; - -class SequentialFileImpl : public SequentialFile { - public: - explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) { - file_->Ref(); - } - - ~SequentialFileImpl() { - file_->Unref(); - } - - virtual Status Read(size_t n, Slice* result, char* scratch) override { - Status s = file_->Read(pos_, n, result, scratch); - if (s.ok()) { - pos_ += result->size(); - } - return s; - } - - virtual Status Skip(uint64_t n) override { - if (pos_ > file_->Size()) { - return Status::IOError("pos_ > file_->Size()"); - } - const size_t available = file_->Size() - pos_; - if (n > available) { - n = available; - } - pos_ += n; - return Status::OK(); - } - - private: - FileState* file_; - size_t pos_; -}; - -class RandomAccessFileImpl : public RandomAccessFile { - public: - explicit RandomAccessFileImpl(FileState* file) : file_(file) { - file_->Ref(); - } - - ~RandomAccessFileImpl() { - file_->Unref(); - } - - virtual Status Read(uint64_t offset, size_t n, Slice* result, - char* scratch) const override { - return file_->Read(offset, n, result, scratch); - } - - private: - FileState* file_; -}; - -class WritableFileImpl : public WritableFile { - public: - explicit WritableFileImpl(FileState* file) : file_(file) { file_->Ref(); } - - ~WritableFileImpl() { - file_->Unref(); - } - - virtual Status Append(const Slice& data) override { - return file_->Append(data); - } - virtual Status Truncate(uint64_t size) override { - return Status::OK(); - } - virtual Status Close() override { return Status::OK(); } - virtual Status Flush() override { return Status::OK(); } - virtual Status Sync() override { return Status::OK(); } - - private: - FileState* file_; -}; - -class InMemoryDirectory : public Directory { - public: - virtual Status Fsync() override { return Status::OK(); } -}; - -class InMemoryEnv : public EnvWrapper { - public: - explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { } - - virtual ~InMemoryEnv() { - for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){ - i->second->Unref(); - } - } - - // Partial implementation of the Env interface. - virtual Status NewSequentialFile(const std::string& fname, - unique_ptr* result, - const EnvOptions& soptions) override { - std::string nfname = NormalizeFileName(fname); - MutexLock lock(&mutex_); - if (file_map_.find(fname) == file_map_.end()) { - *result = NULL; - return Status::IOError(fname, "File not found"); - } - - result->reset(new SequentialFileImpl(file_map_[nfname])); - return Status::OK(); - } - - virtual Status NewRandomAccessFile(const std::string& fname, - unique_ptr* result, - const EnvOptions& soptions) override { - std::string nfname = NormalizeFileName(fname); - MutexLock lock(&mutex_); - if (file_map_.find(nfname) == file_map_.end()) { - *result = NULL; - return Status::IOError(fname, "File not found"); - } - - result->reset(new RandomAccessFileImpl(file_map_[nfname])); - return Status::OK(); - } - - virtual Status NewWritableFile(const std::string& fname, - unique_ptr* result, - const EnvOptions& soptions) override { - std::string nfname = NormalizeFileName(fname); - MutexLock lock(&mutex_); - if (file_map_.find(nfname) != file_map_.end()) { - DeleteFileInternal(nfname); - } - - FileState* file = new FileState(); - file->Ref(); - file_map_[nfname] = file; - - result->reset(new WritableFileImpl(file)); - return Status::OK(); - } - - virtual Status NewDirectory(const std::string& name, - unique_ptr* result) override { - result->reset(new InMemoryDirectory()); - return Status::OK(); - } - - virtual Status FileExists(const std::string& fname) override { - std::string nfname = NormalizeFileName(fname); - MutexLock lock(&mutex_); - if (file_map_.find(nfname) != file_map_.end()) { - return Status::OK(); - } else { - return Status::NotFound(); - } - } - - virtual Status GetChildren(const std::string& dir, - std::vector* result) override { - MutexLock lock(&mutex_); - result->clear(); - - bool found_dir = false; - for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){ - const std::string& filename = i->first; - - if (dir == filename) { - found_dir = true; - } else if (filename.size() >= dir.size() + 1 && - filename[dir.size()] == '/' && - Slice(filename).starts_with(Slice(dir))) { - found_dir = true; - result->push_back(filename.substr(dir.size() + 1)); - } - } - - return found_dir ? Status::OK() : Status::NotFound(); - } - - void DeleteFileInternal(const std::string& fname) { - if (file_map_.find(fname) == file_map_.end()) { - return; - } - - file_map_[fname]->Unref(); - file_map_.erase(fname); - } - - virtual Status DeleteFile(const std::string& fname) override { - std::string nfname = NormalizeFileName(fname); - MutexLock lock(&mutex_); - if (file_map_.find(nfname) == file_map_.end()) { - return Status::IOError(fname, "File not found"); - } - - DeleteFileInternal(nfname); - return Status::OK(); - } - - virtual Status CreateDir(const std::string& dirname) override { - auto ndirname = NormalizeFileName(dirname); - if (file_map_.find(ndirname) == file_map_.end()) { - FileState* file = new FileState(); - file->Ref(); - file_map_[ndirname] = file; - } else { - return Status::IOError(); - } - return Status::OK(); - } - - virtual Status CreateDirIfMissing(const std::string& dirname) override { - CreateDir(dirname); - return Status::OK(); - } - - virtual Status DeleteDir(const std::string& dirname) override { - return DeleteFile(dirname); - } - - virtual Status GetFileSize(const std::string& fname, - uint64_t* file_size) override { - std::string nfname = NormalizeFileName(fname); - MutexLock lock(&mutex_); - - if (file_map_.find(nfname) == file_map_.end()) { - return Status::IOError(fname, "File not found"); - } - - *file_size = file_map_[nfname]->Size(); - return Status::OK(); - } - - virtual Status GetFileModificationTime(const std::string& fname, - uint64_t* time) override { - return Status::NotSupported("getFileMTime", "Not supported in MemEnv"); - } - - virtual Status RenameFile(const std::string& src, - const std::string& dest) override { - std::string nsrc = NormalizeFileName(src); - std::string ndest = NormalizeFileName(dest); - MutexLock lock(&mutex_); - if (file_map_.find(nsrc) == file_map_.end()) { - return Status::IOError(src, "File not found"); - } - - DeleteFileInternal(dest); - file_map_[ndest] = file_map_[nsrc]; - file_map_.erase(nsrc); - return Status::OK(); - } - - virtual Status LockFile(const std::string& fname, FileLock** lock) override { - *lock = new FileLock; - return Status::OK(); - } - - virtual Status UnlockFile(FileLock* lock) override { - delete lock; - return Status::OK(); - } - - virtual Status GetTestDirectory(std::string* path) override { - *path = "/test"; - return Status::OK(); - } - - private: - // Map from filenames to FileState objects, representing a simple file system. - typedef std::map FileSystem; - port::Mutex mutex_; - FileSystem file_map_; // Protected by mutex_. -}; - -} // namespace - -Env* NewMemEnv(Env* base_env) { - return new InMemoryEnv(base_env); -} - -#else // ROCKSDB_LITE - -Env* NewMemEnv(Env* base_env) { return nullptr; } - -#endif // !ROCKSDB_LITE - -} // namespace rocksdb diff --git a/env/mock_env.cc b/env/mock_env.cc index 2e612c58f..8868b358d 100644 --- a/env/mock_env.cc +++ b/env/mock_env.cc @@ -782,4 +782,14 @@ void MockEnv::FakeSleepForMicroseconds(int64_t micros) { fake_sleep_micros_.fetch_add(micros); } +#ifndef ROCKSDB_LITE +// This is to maintain the behavior before swithcing from InMemoryEnv to MockEnv +Env* NewMemEnv(Env* base_env) { return new MockEnv(base_env); } + +#else // ROCKSDB_LITE + +Env* NewMemEnv(Env* base_env) { return nullptr; } + +#endif // !ROCKSDB_LITE + } // namespace rocksdb diff --git a/src.mk b/src.mk index b4ee808f2..bd362c641 100644 --- a/src.mk +++ b/src.mk @@ -60,7 +60,7 @@ LIB_SOURCES = \ env/env_hdfs.cc \ env/env_posix.cc \ env/io_posix.cc \ - env/memenv.cc \ + env/mock_env.cc \ memtable/hash_cuckoo_rep.cc \ memtable/hash_linklist_rep.cc \ memtable/hash_skiplist_rep.cc \ @@ -206,7 +206,6 @@ TOOL_LIB_SOURCES = \ utilities/blob_db/blob_dump_tool.cc \ MOCK_LIB_SOURCES = \ - env/mock_env.cc \ table/mock_table.cc \ util/fault_injection_test_env.cc