Summary: TestMemEnv simulates all Env APIs using in-memory data structures. We can use it to speed up db_test run, which is now reduced ~7mins when it is enabled. We can also add features to simulate power/disk failures in the next step TestMemEnv is derived from helper/mem_env mem_env can not be used for rocksdb since some of its APIs do not give the same results as env_posix. And its file read/write is not thread safe Test Plan: make all -j32 ./db_test ./env_mem_test Reviewers: sdong, yhchiang, rven, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D28035main
Lei Jin
10 years ago
parent
8db24f4b35
commit
0e526eb9d7
@ -0,0 +1,607 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include "util/mock_env.h" |
||||||
|
#include <sys/time.h> |
||||||
|
#include <algorithm> |
||||||
|
#include <chrono> |
||||||
|
#include "util/rate_limiter.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class MemFile { |
||||||
|
public: |
||||||
|
explicit MemFile(const std::string& fn) : |
||||||
|
fn_(fn), refs_(0), size_(0), modified_time_(Now()) {} |
||||||
|
|
||||||
|
void Ref() { |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
++refs_; |
||||||
|
} |
||||||
|
|
||||||
|
void Unref() { |
||||||
|
bool do_delete = false; |
||||||
|
{ |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
--refs_; |
||||||
|
assert(refs_ >= 0); |
||||||
|
if (refs_ <= 0) { |
||||||
|
do_delete = true; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (do_delete) { |
||||||
|
delete this; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
uint64_t Size() const { |
||||||
|
return size_; |
||||||
|
} |
||||||
|
|
||||||
|
void Truncate(size_t size) { |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
if (size < size_) { |
||||||
|
data_.resize(size); |
||||||
|
size_ = size; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
if (offset > Size()) { |
||||||
|
return Status::IOError("Offset greater than file size."); |
||||||
|
} |
||||||
|
const uint64_t available = Size() - offset; |
||||||
|
if (n > available) { |
||||||
|
n = available; |
||||||
|
} |
||||||
|
if (n == 0) { |
||||||
|
*result = Slice(); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
if (scratch) { |
||||||
|
memcpy(scratch, &(data_[offset]), n); |
||||||
|
*result = Slice(scratch, n); |
||||||
|
} else { |
||||||
|
*result = Slice(&(data_[offset]), n); |
||||||
|
} |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status Append(const Slice& data) { |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
data_.append(data.data(), data.size()); |
||||||
|
size_ = data_.size(); |
||||||
|
modified_time_ = Now(); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status Fsync() { |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
uint64_t ModifiedTime() const { |
||||||
|
return modified_time_; |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
uint64_t Now() { |
||||||
|
return std::chrono::duration_cast<std::chrono::seconds>( |
||||||
|
std::chrono::system_clock::now().time_since_epoch()).count(); |
||||||
|
} |
||||||
|
|
||||||
|
// Private since only Unref() should be used to delete it.
|
||||||
|
~MemFile() { |
||||||
|
assert(refs_ == 0); |
||||||
|
} |
||||||
|
|
||||||
|
// No copying allowed.
|
||||||
|
MemFile(const MemFile&); |
||||||
|
void operator=(const MemFile&); |
||||||
|
|
||||||
|
const std::string fn_; |
||||||
|
mutable port::Mutex mutex_; |
||||||
|
int refs_; |
||||||
|
|
||||||
|
std::string data_; |
||||||
|
std::atomic<uint64_t> size_; |
||||||
|
std::atomic<uint64_t> modified_time_; |
||||||
|
}; |
||||||
|
|
||||||
|
namespace { |
||||||
|
|
||||||
|
class SequentialFileImpl : public SequentialFile { |
||||||
|
public: |
||||||
|
explicit SequentialFileImpl(MemFile* file) : file_(file), pos_(0) { |
||||||
|
file_->Ref(); |
||||||
|
} |
||||||
|
|
||||||
|
~SequentialFileImpl() { |
||||||
|
file_->Unref(); |
||||||
|
} |
||||||
|
|
||||||
|
virtual Status Read(size_t n, Slice* result, char* scratch) { |
||||||
|
Status s = file_->Read(pos_, n, result, scratch); |
||||||
|
if (s.ok()) { |
||||||
|
pos_ += result->size(); |
||||||
|
} |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
virtual Status Skip(uint64_t n) { |
||||||
|
if (pos_ > file_->Size()) { |
||||||
|
return Status::IOError("pos_ > file_->Size()"); |
||||||
|
} |
||||||
|
const size_t available = file_->Size() - pos_; |
||||||
|
if (n > available) { |
||||||
|
n = available; |
||||||
|
} |
||||||
|
pos_ += n; |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
MemFile* file_; |
||||||
|
size_t pos_; |
||||||
|
}; |
||||||
|
|
||||||
|
class RandomAccessFileImpl : public RandomAccessFile { |
||||||
|
public: |
||||||
|
explicit RandomAccessFileImpl(MemFile* file) : file_(file) { |
||||||
|
file_->Ref(); |
||||||
|
} |
||||||
|
|
||||||
|
~RandomAccessFileImpl() { |
||||||
|
file_->Unref(); |
||||||
|
} |
||||||
|
|
||||||
|
virtual Status Read(uint64_t offset, size_t n, Slice* result, |
||||||
|
char* scratch) const { |
||||||
|
return file_->Read(offset, n, result, scratch); |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
MemFile* file_; |
||||||
|
}; |
||||||
|
|
||||||
|
class WritableFileImpl : public WritableFile { |
||||||
|
public: |
||||||
|
WritableFileImpl(MemFile* file, RateLimiter* rate_limiter) |
||||||
|
: file_(file), |
||||||
|
rate_limiter_(rate_limiter) { |
||||||
|
file_->Ref(); |
||||||
|
} |
||||||
|
|
||||||
|
~WritableFileImpl() { |
||||||
|
file_->Unref(); |
||||||
|
} |
||||||
|
|
||||||
|
virtual Status Append(const Slice& data) { |
||||||
|
uint64_t bytes_written = 0; |
||||||
|
while (bytes_written < data.size()) { |
||||||
|
auto bytes = RequestToken(data.size() - bytes_written); |
||||||
|
Status s = file_->Append(Slice(data.data() + bytes_written, bytes)); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
bytes_written += bytes; |
||||||
|
} |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
virtual Status Close() { |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
virtual Status Flush() { |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
virtual Status Sync() { |
||||||
|
return file_->Fsync(); |
||||||
|
} |
||||||
|
|
||||||
|
virtual uint64_t GetFileSize() { |
||||||
|
return file_->Size(); |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
inline size_t RequestToken(size_t bytes) { |
||||||
|
if (rate_limiter_ && io_priority_ < Env::IO_TOTAL) { |
||||||
|
bytes = std::min(bytes, |
||||||
|
static_cast<size_t>(rate_limiter_->GetSingleBurstBytes())); |
||||||
|
rate_limiter_->Request(bytes, io_priority_); |
||||||
|
} |
||||||
|
return bytes; |
||||||
|
} |
||||||
|
|
||||||
|
MemFile* file_; |
||||||
|
RateLimiter* rate_limiter_; |
||||||
|
}; |
||||||
|
|
||||||
|
class MockEnvDirectory : public Directory { |
||||||
|
public: |
||||||
|
virtual Status Fsync() { return Status::OK(); } |
||||||
|
}; |
||||||
|
|
||||||
|
class MockEnvFileLock : public FileLock { |
||||||
|
public: |
||||||
|
explicit MockEnvFileLock(const std::string& fname) |
||||||
|
: fname_(fname) {} |
||||||
|
|
||||||
|
std::string FileName() const { |
||||||
|
return fname_; |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
const std::string fname_; |
||||||
|
}; |
||||||
|
|
||||||
|
class TestMemLogger : public Logger { |
||||||
|
private: |
||||||
|
std::unique_ptr<WritableFile> file_; |
||||||
|
std::atomic_size_t log_size_; |
||||||
|
static const uint64_t flush_every_seconds_ = 5; |
||||||
|
std::atomic_uint_fast64_t last_flush_micros_; |
||||||
|
Env* env_; |
||||||
|
bool flush_pending_; |
||||||
|
|
||||||
|
public: |
||||||
|
TestMemLogger(std::unique_ptr<WritableFile> f, Env* env, |
||||||
|
const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL) |
||||||
|
: Logger(log_level), |
||||||
|
file_(std::move(f)), |
||||||
|
log_size_(0), |
||||||
|
last_flush_micros_(0), |
||||||
|
env_(env), |
||||||
|
flush_pending_(false) {} |
||||||
|
virtual ~TestMemLogger() { |
||||||
|
} |
||||||
|
|
||||||
|
virtual void Flush() { |
||||||
|
if (flush_pending_) { |
||||||
|
flush_pending_ = false; |
||||||
|
} |
||||||
|
last_flush_micros_ = env_->NowMicros(); |
||||||
|
} |
||||||
|
virtual void Logv(const char* format, va_list ap) { |
||||||
|
// We try twice: the first time with a fixed-size stack allocated buffer,
|
||||||
|
// and the second time with a much larger dynamically allocated buffer.
|
||||||
|
char buffer[500]; |
||||||
|
for (int iter = 0; iter < 2; iter++) { |
||||||
|
char* base; |
||||||
|
int bufsize; |
||||||
|
if (iter == 0) { |
||||||
|
bufsize = sizeof(buffer); |
||||||
|
base = buffer; |
||||||
|
} else { |
||||||
|
bufsize = 30000; |
||||||
|
base = new char[bufsize]; |
||||||
|
} |
||||||
|
char* p = base; |
||||||
|
char* limit = base + bufsize; |
||||||
|
|
||||||
|
struct timeval now_tv; |
||||||
|
gettimeofday(&now_tv, nullptr); |
||||||
|
const time_t seconds = now_tv.tv_sec; |
||||||
|
struct tm t; |
||||||
|
localtime_r(&seconds, &t); |
||||||
|
p += snprintf(p, limit - p, |
||||||
|
"%04d/%02d/%02d-%02d:%02d:%02d.%06d ", |
||||||
|
t.tm_year + 1900, |
||||||
|
t.tm_mon + 1, |
||||||
|
t.tm_mday, |
||||||
|
t.tm_hour, |
||||||
|
t.tm_min, |
||||||
|
t.tm_sec, |
||||||
|
static_cast<int>(now_tv.tv_usec)); |
||||||
|
|
||||||
|
// Print the message
|
||||||
|
if (p < limit) { |
||||||
|
va_list backup_ap; |
||||||
|
va_copy(backup_ap, ap); |
||||||
|
p += vsnprintf(p, limit - p, format, backup_ap); |
||||||
|
va_end(backup_ap); |
||||||
|
} |
||||||
|
|
||||||
|
// Truncate to available space if necessary
|
||||||
|
if (p >= limit) { |
||||||
|
if (iter == 0) { |
||||||
|
continue; // Try again with larger buffer
|
||||||
|
} else { |
||||||
|
p = limit - 1; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Add newline if necessary
|
||||||
|
if (p == base || p[-1] != '\n') { |
||||||
|
*p++ = '\n'; |
||||||
|
} |
||||||
|
|
||||||
|
assert(p <= limit); |
||||||
|
const size_t write_size = p - base; |
||||||
|
|
||||||
|
file_->Append(Slice(base, write_size)); |
||||||
|
flush_pending_ = true; |
||||||
|
log_size_ += write_size; |
||||||
|
uint64_t now_micros = static_cast<uint64_t>(now_tv.tv_sec) * 1000000 + |
||||||
|
now_tv.tv_usec; |
||||||
|
if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) { |
||||||
|
flush_pending_ = false; |
||||||
|
last_flush_micros_ = now_micros; |
||||||
|
} |
||||||
|
if (base != buffer) { |
||||||
|
delete[] base; |
||||||
|
} |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
size_t GetLogFileSize() const { |
||||||
|
return log_size_; |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
} // Anonymous namespace
|
||||||
|
|
||||||
|
MockEnv::MockEnv(Env* base_env) |
||||||
|
: EnvWrapper(base_env) {} |
||||||
|
|
||||||
|
MockEnv::~MockEnv() { |
||||||
|
for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i) { |
||||||
|
i->second->Unref(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Partial implementation of the Env interface.
|
||||||
|
Status MockEnv::NewSequentialFile(const std::string& fname, |
||||||
|
unique_ptr<SequentialFile>* result, |
||||||
|
const EnvOptions& soptions) { |
||||||
|
auto fn = NormalizePath(fname); |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
if (file_map_.find(fn) == file_map_.end()) { |
||||||
|
*result = NULL; |
||||||
|
return Status::IOError(fn, "File not found"); |
||||||
|
} |
||||||
|
auto* f = file_map_[fn]; |
||||||
|
result->reset(new SequentialFileImpl(f)); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::NewRandomAccessFile(const std::string& fname, |
||||||
|
unique_ptr<RandomAccessFile>* result, |
||||||
|
const EnvOptions& soptions) { |
||||||
|
auto fn = NormalizePath(fname); |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
if (file_map_.find(fn) == file_map_.end()) { |
||||||
|
*result = NULL; |
||||||
|
return Status::IOError(fn, "File not found"); |
||||||
|
} |
||||||
|
auto* f = file_map_[fn]; |
||||||
|
result->reset(new RandomAccessFileImpl(f)); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::NewWritableFile(const std::string& fname, |
||||||
|
unique_ptr<WritableFile>* result, |
||||||
|
const EnvOptions& env_options) { |
||||||
|
auto fn = NormalizePath(fname); |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
if (file_map_.find(fn) != file_map_.end()) { |
||||||
|
DeleteFileInternal(fn); |
||||||
|
} |
||||||
|
MemFile* file = new MemFile(fn); |
||||||
|
file->Ref(); |
||||||
|
file_map_[fn] = file; |
||||||
|
|
||||||
|
result->reset(new WritableFileImpl(file, env_options.rate_limiter)); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::NewRandomRWFile(const std::string& fname, |
||||||
|
unique_ptr<RandomRWFile>* result, |
||||||
|
const EnvOptions& options) { |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::NewDirectory(const std::string& name, |
||||||
|
unique_ptr<Directory>* result) { |
||||||
|
result->reset(new MockEnvDirectory()); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
bool MockEnv::FileExists(const std::string& fname) { |
||||||
|
auto fn = NormalizePath(fname); |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
if (file_map_.find(fn) != file_map_.end()) { |
||||||
|
// File exists
|
||||||
|
return true; |
||||||
|
} |
||||||
|
// Now also check if fn exists as a dir
|
||||||
|
for (const auto& iter : file_map_) { |
||||||
|
const std::string& filename = iter.first; |
||||||
|
if (filename.size() >= fn.size() + 1 && |
||||||
|
filename[fn.size()] == '/' && |
||||||
|
Slice(filename).starts_with(Slice(fn))) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::GetChildren(const std::string& dir, |
||||||
|
std::vector<std::string>* result) { |
||||||
|
auto d = NormalizePath(dir); |
||||||
|
{ |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
result->clear(); |
||||||
|
for (const auto& iter : file_map_) { |
||||||
|
const std::string& filename = iter.first; |
||||||
|
|
||||||
|
if (filename.size() >= d.size() + 1 && filename[d.size()] == '/' && |
||||||
|
Slice(filename).starts_with(Slice(d))) { |
||||||
|
size_t next_slash = filename.find('/', d.size() + 1); |
||||||
|
if (next_slash != std::string::npos) { |
||||||
|
result->push_back(filename.substr( |
||||||
|
d.size() + 1, next_slash - d.size() - 1)); |
||||||
|
} else { |
||||||
|
result->push_back(filename.substr(d.size() + 1)); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
result->erase(std::unique(result->begin(), result->end()), result->end()); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
void MockEnv::DeleteFileInternal(const std::string& fname) { |
||||||
|
assert(fname == NormalizePath(fname)); |
||||||
|
if (file_map_.find(fname) == file_map_.end()) { |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
file_map_[fname]->Unref(); |
||||||
|
file_map_.erase(fname); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::DeleteFile(const std::string& fname) { |
||||||
|
auto fn = NormalizePath(fname); |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
if (file_map_.find(fn) == file_map_.end()) { |
||||||
|
return Status::IOError(fn, "File not found"); |
||||||
|
} |
||||||
|
|
||||||
|
DeleteFileInternal(fn); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::CreateDir(const std::string& dirname) { |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::CreateDirIfMissing(const std::string& dirname) { |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::DeleteDir(const std::string& dirname) { |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::GetFileSize(const std::string& fname, uint64_t* file_size) { |
||||||
|
auto fn = NormalizePath(fname); |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
auto iter = file_map_.find(fn); |
||||||
|
if (iter == file_map_.end()) { |
||||||
|
return Status::IOError(fn, "File not found"); |
||||||
|
} |
||||||
|
|
||||||
|
*file_size = iter->second->Size(); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::GetFileModificationTime(const std::string& fname, |
||||||
|
uint64_t* time) { |
||||||
|
auto fn = NormalizePath(fname); |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
auto iter = file_map_.find(fn); |
||||||
|
if (iter == file_map_.end()) { |
||||||
|
return Status::IOError(fn, "File not found"); |
||||||
|
} |
||||||
|
*time = iter->second->ModifiedTime(); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::RenameFile(const std::string& src, |
||||||
|
const std::string& target) { |
||||||
|
auto s = NormalizePath(src); |
||||||
|
auto t = NormalizePath(target); |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
if (file_map_.find(s) == file_map_.end()) { |
||||||
|
return Status::IOError(s, "File not found"); |
||||||
|
} |
||||||
|
|
||||||
|
DeleteFileInternal(t); |
||||||
|
file_map_[t] = file_map_[s]; |
||||||
|
file_map_.erase(s); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::NewLogger(const std::string& fname, |
||||||
|
shared_ptr<Logger>* result) { |
||||||
|
auto fn = NormalizePath(fname); |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
auto iter = file_map_.find(fn); |
||||||
|
MemFile* file = nullptr; |
||||||
|
if (iter == file_map_.end()) { |
||||||
|
file = new MemFile(fn); |
||||||
|
file->Ref(); |
||||||
|
file_map_[fn] = file; |
||||||
|
} else { |
||||||
|
file = iter->second; |
||||||
|
} |
||||||
|
std::unique_ptr<WritableFile> f(new WritableFileImpl(file, nullptr)); |
||||||
|
result->reset(new TestMemLogger(std::move(f), this)); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::LockFile(const std::string& fname, FileLock** flock) { |
||||||
|
auto fn = NormalizePath(fname); |
||||||
|
{ |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
if (file_map_.find(fn) != file_map_.end()) { |
||||||
|
return Status::IOError(fn, "Lock file exists"); |
||||||
|
} |
||||||
|
file_map_[fn] = nullptr; |
||||||
|
} |
||||||
|
*flock = new MockEnvFileLock(fn); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::UnlockFile(FileLock* flock) { |
||||||
|
std::string fn = dynamic_cast<MockEnvFileLock*>(flock)->FileName(); |
||||||
|
{ |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
auto iter = file_map_.find(fn); |
||||||
|
if (iter != file_map_.end()) { |
||||||
|
file_map_.erase(fn); |
||||||
|
} |
||||||
|
} |
||||||
|
delete flock; |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status MockEnv::GetTestDirectory(std::string* path) { |
||||||
|
*path = "/test"; |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
// Non-virtual functions, specific to MockEnv
|
||||||
|
Status MockEnv::Truncate(const std::string& fname, size_t size) { |
||||||
|
auto fn = NormalizePath(fname); |
||||||
|
MutexLock lock(&mutex_); |
||||||
|
auto iter = file_map_.find(fn); |
||||||
|
if (iter == file_map_.end()) { |
||||||
|
return Status::IOError(fn, "File not found"); |
||||||
|
} |
||||||
|
iter->second->Truncate(size); |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
std::string MockEnv::NormalizePath(const std::string path) { |
||||||
|
std::string dst; |
||||||
|
for (auto c : path) { |
||||||
|
if (!dst.empty() && c == '/' && dst.back() == '/') { |
||||||
|
continue; |
||||||
|
} |
||||||
|
dst.push_back(c); |
||||||
|
} |
||||||
|
return dst; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,93 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <atomic> |
||||||
|
#include <map> |
||||||
|
#include <string> |
||||||
|
#include <vector> |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
#include "port/port.h" |
||||||
|
#include "util/mutexlock.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class MemFile; |
||||||
|
class MockEnv : public EnvWrapper { |
||||||
|
public: |
||||||
|
explicit MockEnv(Env* base_env); |
||||||
|
|
||||||
|
virtual ~MockEnv(); |
||||||
|
|
||||||
|
// Partial implementation of the Env interface.
|
||||||
|
virtual Status NewSequentialFile(const std::string& fname, |
||||||
|
unique_ptr<SequentialFile>* result, |
||||||
|
const EnvOptions& soptions); |
||||||
|
|
||||||
|
virtual Status NewRandomAccessFile(const std::string& fname, |
||||||
|
unique_ptr<RandomAccessFile>* result, |
||||||
|
const EnvOptions& soptions); |
||||||
|
|
||||||
|
virtual Status NewWritableFile(const std::string& fname, |
||||||
|
unique_ptr<WritableFile>* result, |
||||||
|
const EnvOptions& env_options); |
||||||
|
|
||||||
|
virtual Status NewRandomRWFile(const std::string& fname, |
||||||
|
unique_ptr<RandomRWFile>* result, |
||||||
|
const EnvOptions& options); |
||||||
|
|
||||||
|
virtual Status NewDirectory(const std::string& name, |
||||||
|
unique_ptr<Directory>* result); |
||||||
|
|
||||||
|
virtual bool FileExists(const std::string& fname); |
||||||
|
|
||||||
|
virtual Status GetChildren(const std::string& dir, |
||||||
|
std::vector<std::string>* result); |
||||||
|
|
||||||
|
void DeleteFileInternal(const std::string& fname); |
||||||
|
|
||||||
|
virtual Status DeleteFile(const std::string& fname); |
||||||
|
|
||||||
|
virtual Status CreateDir(const std::string& dirname); |
||||||
|
|
||||||
|
virtual Status CreateDirIfMissing(const std::string& dirname); |
||||||
|
|
||||||
|
virtual Status DeleteDir(const std::string& dirname); |
||||||
|
|
||||||
|
virtual Status GetFileSize(const std::string& fname, uint64_t* file_size); |
||||||
|
|
||||||
|
virtual Status GetFileModificationTime(const std::string& fname, |
||||||
|
uint64_t* time); |
||||||
|
|
||||||
|
virtual Status RenameFile(const std::string& src, |
||||||
|
const std::string& target); |
||||||
|
|
||||||
|
virtual Status NewLogger(const std::string& fname, |
||||||
|
shared_ptr<Logger>* result); |
||||||
|
|
||||||
|
virtual Status LockFile(const std::string& fname, FileLock** flock); |
||||||
|
|
||||||
|
virtual Status UnlockFile(FileLock* flock); |
||||||
|
|
||||||
|
virtual Status GetTestDirectory(std::string* path); |
||||||
|
|
||||||
|
// Non-virtual functions, specific to MockEnv
|
||||||
|
Status Truncate(const std::string& fname, size_t size); |
||||||
|
|
||||||
|
private: |
||||||
|
std::string NormalizePath(const std::string path); |
||||||
|
|
||||||
|
// Map from filenames to MemFile objects, representing a simple file system.
|
||||||
|
typedef std::map<std::string, MemFile*> FileSystem; |
||||||
|
port::Mutex mutex_; |
||||||
|
FileSystem file_map_; // Protected by mutex_.
|
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,232 @@ |
|||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
#include <vector> |
||||||
|
#include "util/mock_env.h" |
||||||
|
#include "db/db_impl.h" |
||||||
|
#include "rocksdb/db.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "util/testharness.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class MockEnvTest { |
||||||
|
public: |
||||||
|
Env* env_; |
||||||
|
const EnvOptions soptions_; |
||||||
|
|
||||||
|
MockEnvTest() |
||||||
|
: env_(new MockEnv(Env::Default())) { |
||||||
|
} |
||||||
|
~MockEnvTest() { |
||||||
|
delete env_; |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST(MockEnvTest, Basics) { |
||||||
|
uint64_t file_size; |
||||||
|
unique_ptr<WritableFile> writable_file; |
||||||
|
std::vector<std::string> children; |
||||||
|
|
||||||
|
ASSERT_OK(env_->CreateDir("/dir")); |
||||||
|
|
||||||
|
// Check that the directory is empty.
|
||||||
|
ASSERT_TRUE(!env_->FileExists("/dir/non_existent")); |
||||||
|
ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok()); |
||||||
|
ASSERT_OK(env_->GetChildren("/dir", &children)); |
||||||
|
ASSERT_EQ(0U, children.size()); |
||||||
|
|
||||||
|
// Create a file.
|
||||||
|
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_)); |
||||||
|
writable_file.reset(); |
||||||
|
|
||||||
|
// Check that the file exists.
|
||||||
|
ASSERT_TRUE(env_->FileExists("/dir/f")); |
||||||
|
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); |
||||||
|
ASSERT_EQ(0U, file_size); |
||||||
|
ASSERT_OK(env_->GetChildren("/dir", &children)); |
||||||
|
ASSERT_EQ(1U, children.size()); |
||||||
|
ASSERT_EQ("f", children[0]); |
||||||
|
|
||||||
|
// Write to the file.
|
||||||
|
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_)); |
||||||
|
ASSERT_OK(writable_file->Append("abc")); |
||||||
|
writable_file.reset(); |
||||||
|
|
||||||
|
// Check for expected size.
|
||||||
|
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); |
||||||
|
ASSERT_EQ(3U, file_size); |
||||||
|
|
||||||
|
// Check that renaming works.
|
||||||
|
ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok()); |
||||||
|
ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g")); |
||||||
|
ASSERT_TRUE(!env_->FileExists("/dir/f")); |
||||||
|
ASSERT_TRUE(env_->FileExists("/dir/g")); |
||||||
|
ASSERT_OK(env_->GetFileSize("/dir/g", &file_size)); |
||||||
|
ASSERT_EQ(3U, file_size); |
||||||
|
|
||||||
|
// Check that opening non-existent file fails.
|
||||||
|
unique_ptr<SequentialFile> seq_file; |
||||||
|
unique_ptr<RandomAccessFile> rand_file; |
||||||
|
ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file, |
||||||
|
soptions_).ok()); |
||||||
|
ASSERT_TRUE(!seq_file); |
||||||
|
ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file, |
||||||
|
soptions_).ok()); |
||||||
|
ASSERT_TRUE(!rand_file); |
||||||
|
|
||||||
|
// Check that deleting works.
|
||||||
|
ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok()); |
||||||
|
ASSERT_OK(env_->DeleteFile("/dir/g")); |
||||||
|
ASSERT_TRUE(!env_->FileExists("/dir/g")); |
||||||
|
ASSERT_OK(env_->GetChildren("/dir", &children)); |
||||||
|
ASSERT_EQ(0U, children.size()); |
||||||
|
ASSERT_OK(env_->DeleteDir("/dir")); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(MockEnvTest, ReadWrite) { |
||||||
|
unique_ptr<WritableFile> writable_file; |
||||||
|
unique_ptr<SequentialFile> seq_file; |
||||||
|
unique_ptr<RandomAccessFile> rand_file; |
||||||
|
Slice result; |
||||||
|
char scratch[100]; |
||||||
|
|
||||||
|
ASSERT_OK(env_->CreateDir("/dir")); |
||||||
|
|
||||||
|
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_)); |
||||||
|
ASSERT_OK(writable_file->Append("hello ")); |
||||||
|
ASSERT_OK(writable_file->Append("world")); |
||||||
|
writable_file.reset(); |
||||||
|
|
||||||
|
// Read sequentially.
|
||||||
|
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file, soptions_)); |
||||||
|
ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
|
||||||
|
ASSERT_EQ(0, result.compare("hello")); |
||||||
|
ASSERT_OK(seq_file->Skip(1)); |
||||||
|
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world".
|
||||||
|
ASSERT_EQ(0, result.compare("world")); |
||||||
|
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF.
|
||||||
|
ASSERT_EQ(0U, result.size()); |
||||||
|
ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file.
|
||||||
|
ASSERT_OK(seq_file->Read(1000, &result, scratch)); |
||||||
|
ASSERT_EQ(0U, result.size()); |
||||||
|
|
||||||
|
// Random reads.
|
||||||
|
ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file, soptions_)); |
||||||
|
ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
|
||||||
|
ASSERT_EQ(0, result.compare("world")); |
||||||
|
ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
|
||||||
|
ASSERT_EQ(0, result.compare("hello")); |
||||||
|
ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d".
|
||||||
|
ASSERT_EQ(0, result.compare("d")); |
||||||
|
|
||||||
|
// Too high offset.
|
||||||
|
ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok()); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(MockEnvTest, Locks) { |
||||||
|
FileLock* lock; |
||||||
|
|
||||||
|
// These are no-ops, but we test they return success.
|
||||||
|
ASSERT_OK(env_->LockFile("some file", &lock)); |
||||||
|
ASSERT_OK(env_->UnlockFile(lock)); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(MockEnvTest, Misc) { |
||||||
|
std::string test_dir; |
||||||
|
ASSERT_OK(env_->GetTestDirectory(&test_dir)); |
||||||
|
ASSERT_TRUE(!test_dir.empty()); |
||||||
|
|
||||||
|
unique_ptr<WritableFile> writable_file; |
||||||
|
ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file, soptions_)); |
||||||
|
|
||||||
|
// These are no-ops, but we test they return success.
|
||||||
|
ASSERT_OK(writable_file->Sync()); |
||||||
|
ASSERT_OK(writable_file->Flush()); |
||||||
|
ASSERT_OK(writable_file->Close()); |
||||||
|
writable_file.reset(); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(MockEnvTest, LargeWrite) { |
||||||
|
const size_t kWriteSize = 300 * 1024; |
||||||
|
char* scratch = new char[kWriteSize * 2]; |
||||||
|
|
||||||
|
std::string write_data; |
||||||
|
for (size_t i = 0; i < kWriteSize; ++i) { |
||||||
|
write_data.append(1, static_cast<char>(i)); |
||||||
|
} |
||||||
|
|
||||||
|
unique_ptr<WritableFile> writable_file; |
||||||
|
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_)); |
||||||
|
ASSERT_OK(writable_file->Append("foo")); |
||||||
|
ASSERT_OK(writable_file->Append(write_data)); |
||||||
|
writable_file.reset(); |
||||||
|
|
||||||
|
unique_ptr<SequentialFile> seq_file; |
||||||
|
Slice result; |
||||||
|
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file, soptions_)); |
||||||
|
ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
|
||||||
|
ASSERT_EQ(0, result.compare("foo")); |
||||||
|
|
||||||
|
size_t read = 0; |
||||||
|
std::string read_data; |
||||||
|
while (read < kWriteSize) { |
||||||
|
ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch)); |
||||||
|
read_data.append(result.data(), result.size()); |
||||||
|
read += result.size(); |
||||||
|
} |
||||||
|
ASSERT_TRUE(write_data == read_data); |
||||||
|
delete [] scratch; |
||||||
|
} |
||||||
|
|
||||||
|
TEST(MockEnvTest, DBTest) { |
||||||
|
Options options; |
||||||
|
options.create_if_missing = true; |
||||||
|
options.env = env_; |
||||||
|
DB* db; |
||||||
|
|
||||||
|
const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")}; |
||||||
|
const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")}; |
||||||
|
|
||||||
|
ASSERT_OK(DB::Open(options, "/dir/db", &db)); |
||||||
|
for (size_t i = 0; i < 3; ++i) { |
||||||
|
ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i])); |
||||||
|
} |
||||||
|
|
||||||
|
for (size_t i = 0; i < 3; ++i) { |
||||||
|
std::string res; |
||||||
|
ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); |
||||||
|
ASSERT_TRUE(res == vals[i]); |
||||||
|
} |
||||||
|
|
||||||
|
Iterator* iterator = db->NewIterator(ReadOptions()); |
||||||
|
iterator->SeekToFirst(); |
||||||
|
for (size_t i = 0; i < 3; ++i) { |
||||||
|
ASSERT_TRUE(iterator->Valid()); |
||||||
|
ASSERT_TRUE(keys[i] == iterator->key()); |
||||||
|
ASSERT_TRUE(vals[i] == iterator->value()); |
||||||
|
iterator->Next(); |
||||||
|
} |
||||||
|
ASSERT_TRUE(!iterator->Valid()); |
||||||
|
delete iterator; |
||||||
|
|
||||||
|
DBImpl* dbi = reinterpret_cast<DBImpl*>(db); |
||||||
|
ASSERT_OK(dbi->TEST_FlushMemTable()); |
||||||
|
|
||||||
|
for (size_t i = 0; i < 3; ++i) { |
||||||
|
std::string res; |
||||||
|
ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); |
||||||
|
ASSERT_TRUE(res == vals[i]); |
||||||
|
} |
||||||
|
|
||||||
|
delete db; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
return rocksdb::test::RunAllTests(); |
||||||
|
} |
||||||
Loading…
Reference in new issue