Summary: TestMemEnv simulates all Env APIs using in-memory data structures. We can use it to speed up db_test run, which is now reduced ~7mins when it is enabled. We can also add features to simulate power/disk failures in the next step TestMemEnv is derived from helper/mem_env mem_env can not be used for rocksdb since some of its APIs do not give the same results as env_posix. And its file read/write is not thread safe Test Plan: make all -j32 ./db_test ./env_mem_test Reviewers: sdong, yhchiang, rven, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D28035main
Lei Jin
10 years ago
parent
8db24f4b35
commit
0e526eb9d7
@ -0,0 +1,607 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "util/mock_env.h" |
||||
#include <sys/time.h> |
||||
#include <algorithm> |
||||
#include <chrono> |
||||
#include "util/rate_limiter.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class MemFile { |
||||
public: |
||||
explicit MemFile(const std::string& fn) : |
||||
fn_(fn), refs_(0), size_(0), modified_time_(Now()) {} |
||||
|
||||
void Ref() { |
||||
MutexLock lock(&mutex_); |
||||
++refs_; |
||||
} |
||||
|
||||
void Unref() { |
||||
bool do_delete = false; |
||||
{ |
||||
MutexLock lock(&mutex_); |
||||
--refs_; |
||||
assert(refs_ >= 0); |
||||
if (refs_ <= 0) { |
||||
do_delete = true; |
||||
} |
||||
} |
||||
|
||||
if (do_delete) { |
||||
delete this; |
||||
} |
||||
} |
||||
|
||||
uint64_t Size() const { |
||||
return size_; |
||||
} |
||||
|
||||
void Truncate(size_t size) { |
||||
MutexLock lock(&mutex_); |
||||
if (size < size_) { |
||||
data_.resize(size); |
||||
size_ = size; |
||||
} |
||||
} |
||||
|
||||
Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { |
||||
MutexLock lock(&mutex_); |
||||
if (offset > Size()) { |
||||
return Status::IOError("Offset greater than file size."); |
||||
} |
||||
const uint64_t available = Size() - offset; |
||||
if (n > available) { |
||||
n = available; |
||||
} |
||||
if (n == 0) { |
||||
*result = Slice(); |
||||
return Status::OK(); |
||||
} |
||||
if (scratch) { |
||||
memcpy(scratch, &(data_[offset]), n); |
||||
*result = Slice(scratch, n); |
||||
} else { |
||||
*result = Slice(&(data_[offset]), n); |
||||
} |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status Append(const Slice& data) { |
||||
MutexLock lock(&mutex_); |
||||
data_.append(data.data(), data.size()); |
||||
size_ = data_.size(); |
||||
modified_time_ = Now(); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status Fsync() { |
||||
return Status::OK(); |
||||
} |
||||
|
||||
uint64_t ModifiedTime() const { |
||||
return modified_time_; |
||||
} |
||||
|
||||
private: |
||||
uint64_t Now() { |
||||
return std::chrono::duration_cast<std::chrono::seconds>( |
||||
std::chrono::system_clock::now().time_since_epoch()).count(); |
||||
} |
||||
|
||||
// Private since only Unref() should be used to delete it.
|
||||
~MemFile() { |
||||
assert(refs_ == 0); |
||||
} |
||||
|
||||
// No copying allowed.
|
||||
MemFile(const MemFile&); |
||||
void operator=(const MemFile&); |
||||
|
||||
const std::string fn_; |
||||
mutable port::Mutex mutex_; |
||||
int refs_; |
||||
|
||||
std::string data_; |
||||
std::atomic<uint64_t> size_; |
||||
std::atomic<uint64_t> modified_time_; |
||||
}; |
||||
|
||||
namespace { |
||||
|
||||
class SequentialFileImpl : public SequentialFile { |
||||
public: |
||||
explicit SequentialFileImpl(MemFile* file) : file_(file), pos_(0) { |
||||
file_->Ref(); |
||||
} |
||||
|
||||
~SequentialFileImpl() { |
||||
file_->Unref(); |
||||
} |
||||
|
||||
virtual Status Read(size_t n, Slice* result, char* scratch) { |
||||
Status s = file_->Read(pos_, n, result, scratch); |
||||
if (s.ok()) { |
||||
pos_ += result->size(); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
virtual Status Skip(uint64_t n) { |
||||
if (pos_ > file_->Size()) { |
||||
return Status::IOError("pos_ > file_->Size()"); |
||||
} |
||||
const size_t available = file_->Size() - pos_; |
||||
if (n > available) { |
||||
n = available; |
||||
} |
||||
pos_ += n; |
||||
return Status::OK(); |
||||
} |
||||
|
||||
private: |
||||
MemFile* file_; |
||||
size_t pos_; |
||||
}; |
||||
|
||||
class RandomAccessFileImpl : public RandomAccessFile { |
||||
public: |
||||
explicit RandomAccessFileImpl(MemFile* file) : file_(file) { |
||||
file_->Ref(); |
||||
} |
||||
|
||||
~RandomAccessFileImpl() { |
||||
file_->Unref(); |
||||
} |
||||
|
||||
virtual Status Read(uint64_t offset, size_t n, Slice* result, |
||||
char* scratch) const { |
||||
return file_->Read(offset, n, result, scratch); |
||||
} |
||||
|
||||
private: |
||||
MemFile* file_; |
||||
}; |
||||
|
||||
class WritableFileImpl : public WritableFile { |
||||
public: |
||||
WritableFileImpl(MemFile* file, RateLimiter* rate_limiter) |
||||
: file_(file), |
||||
rate_limiter_(rate_limiter) { |
||||
file_->Ref(); |
||||
} |
||||
|
||||
~WritableFileImpl() { |
||||
file_->Unref(); |
||||
} |
||||
|
||||
virtual Status Append(const Slice& data) { |
||||
uint64_t bytes_written = 0; |
||||
while (bytes_written < data.size()) { |
||||
auto bytes = RequestToken(data.size() - bytes_written); |
||||
Status s = file_->Append(Slice(data.data() + bytes_written, bytes)); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
bytes_written += bytes; |
||||
} |
||||
return Status::OK(); |
||||
} |
||||
|
||||
virtual Status Close() { |
||||
return Status::OK(); |
||||
} |
||||
|
||||
virtual Status Flush() { |
||||
return Status::OK(); |
||||
} |
||||
|
||||
virtual Status Sync() { |
||||
return file_->Fsync(); |
||||
} |
||||
|
||||
virtual uint64_t GetFileSize() { |
||||
return file_->Size(); |
||||
} |
||||
|
||||
private: |
||||
inline size_t RequestToken(size_t bytes) { |
||||
if (rate_limiter_ && io_priority_ < Env::IO_TOTAL) { |
||||
bytes = std::min(bytes, |
||||
static_cast<size_t>(rate_limiter_->GetSingleBurstBytes())); |
||||
rate_limiter_->Request(bytes, io_priority_); |
||||
} |
||||
return bytes; |
||||
} |
||||
|
||||
MemFile* file_; |
||||
RateLimiter* rate_limiter_; |
||||
}; |
||||
|
||||
class MockEnvDirectory : public Directory { |
||||
public: |
||||
virtual Status Fsync() { return Status::OK(); } |
||||
}; |
||||
|
||||
class MockEnvFileLock : public FileLock { |
||||
public: |
||||
explicit MockEnvFileLock(const std::string& fname) |
||||
: fname_(fname) {} |
||||
|
||||
std::string FileName() const { |
||||
return fname_; |
||||
} |
||||
|
||||
private: |
||||
const std::string fname_; |
||||
}; |
||||
|
||||
class TestMemLogger : public Logger { |
||||
private: |
||||
std::unique_ptr<WritableFile> file_; |
||||
std::atomic_size_t log_size_; |
||||
static const uint64_t flush_every_seconds_ = 5; |
||||
std::atomic_uint_fast64_t last_flush_micros_; |
||||
Env* env_; |
||||
bool flush_pending_; |
||||
|
||||
public: |
||||
TestMemLogger(std::unique_ptr<WritableFile> f, Env* env, |
||||
const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL) |
||||
: Logger(log_level), |
||||
file_(std::move(f)), |
||||
log_size_(0), |
||||
last_flush_micros_(0), |
||||
env_(env), |
||||
flush_pending_(false) {} |
||||
virtual ~TestMemLogger() { |
||||
} |
||||
|
||||
virtual void Flush() { |
||||
if (flush_pending_) { |
||||
flush_pending_ = false; |
||||
} |
||||
last_flush_micros_ = env_->NowMicros(); |
||||
} |
||||
virtual void Logv(const char* format, va_list ap) { |
||||
// We try twice: the first time with a fixed-size stack allocated buffer,
|
||||
// and the second time with a much larger dynamically allocated buffer.
|
||||
char buffer[500]; |
||||
for (int iter = 0; iter < 2; iter++) { |
||||
char* base; |
||||
int bufsize; |
||||
if (iter == 0) { |
||||
bufsize = sizeof(buffer); |
||||
base = buffer; |
||||
} else { |
||||
bufsize = 30000; |
||||
base = new char[bufsize]; |
||||
} |
||||
char* p = base; |
||||
char* limit = base + bufsize; |
||||
|
||||
struct timeval now_tv; |
||||
gettimeofday(&now_tv, nullptr); |
||||
const time_t seconds = now_tv.tv_sec; |
||||
struct tm t; |
||||
localtime_r(&seconds, &t); |
||||
p += snprintf(p, limit - p, |
||||
"%04d/%02d/%02d-%02d:%02d:%02d.%06d ", |
||||
t.tm_year + 1900, |
||||
t.tm_mon + 1, |
||||
t.tm_mday, |
||||
t.tm_hour, |
||||
t.tm_min, |
||||
t.tm_sec, |
||||
static_cast<int>(now_tv.tv_usec)); |
||||
|
||||
// Print the message
|
||||
if (p < limit) { |
||||
va_list backup_ap; |
||||
va_copy(backup_ap, ap); |
||||
p += vsnprintf(p, limit - p, format, backup_ap); |
||||
va_end(backup_ap); |
||||
} |
||||
|
||||
// Truncate to available space if necessary
|
||||
if (p >= limit) { |
||||
if (iter == 0) { |
||||
continue; // Try again with larger buffer
|
||||
} else { |
||||
p = limit - 1; |
||||
} |
||||
} |
||||
|
||||
// Add newline if necessary
|
||||
if (p == base || p[-1] != '\n') { |
||||
*p++ = '\n'; |
||||
} |
||||
|
||||
assert(p <= limit); |
||||
const size_t write_size = p - base; |
||||
|
||||
file_->Append(Slice(base, write_size)); |
||||
flush_pending_ = true; |
||||
log_size_ += write_size; |
||||
uint64_t now_micros = static_cast<uint64_t>(now_tv.tv_sec) * 1000000 + |
||||
now_tv.tv_usec; |
||||
if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) { |
||||
flush_pending_ = false; |
||||
last_flush_micros_ = now_micros; |
||||
} |
||||
if (base != buffer) { |
||||
delete[] base; |
||||
} |
||||
break; |
||||
} |
||||
} |
||||
size_t GetLogFileSize() const { |
||||
return log_size_; |
||||
} |
||||
}; |
||||
|
||||
} // Anonymous namespace
|
||||
|
||||
MockEnv::MockEnv(Env* base_env) |
||||
: EnvWrapper(base_env) {} |
||||
|
||||
MockEnv::~MockEnv() { |
||||
for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i) { |
||||
i->second->Unref(); |
||||
} |
||||
} |
||||
|
||||
// Partial implementation of the Env interface.
|
||||
Status MockEnv::NewSequentialFile(const std::string& fname, |
||||
unique_ptr<SequentialFile>* result, |
||||
const EnvOptions& soptions) { |
||||
auto fn = NormalizePath(fname); |
||||
MutexLock lock(&mutex_); |
||||
if (file_map_.find(fn) == file_map_.end()) { |
||||
*result = NULL; |
||||
return Status::IOError(fn, "File not found"); |
||||
} |
||||
auto* f = file_map_[fn]; |
||||
result->reset(new SequentialFileImpl(f)); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::NewRandomAccessFile(const std::string& fname, |
||||
unique_ptr<RandomAccessFile>* result, |
||||
const EnvOptions& soptions) { |
||||
auto fn = NormalizePath(fname); |
||||
MutexLock lock(&mutex_); |
||||
if (file_map_.find(fn) == file_map_.end()) { |
||||
*result = NULL; |
||||
return Status::IOError(fn, "File not found"); |
||||
} |
||||
auto* f = file_map_[fn]; |
||||
result->reset(new RandomAccessFileImpl(f)); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::NewWritableFile(const std::string& fname, |
||||
unique_ptr<WritableFile>* result, |
||||
const EnvOptions& env_options) { |
||||
auto fn = NormalizePath(fname); |
||||
MutexLock lock(&mutex_); |
||||
if (file_map_.find(fn) != file_map_.end()) { |
||||
DeleteFileInternal(fn); |
||||
} |
||||
MemFile* file = new MemFile(fn); |
||||
file->Ref(); |
||||
file_map_[fn] = file; |
||||
|
||||
result->reset(new WritableFileImpl(file, env_options.rate_limiter)); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::NewRandomRWFile(const std::string& fname, |
||||
unique_ptr<RandomRWFile>* result, |
||||
const EnvOptions& options) { |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::NewDirectory(const std::string& name, |
||||
unique_ptr<Directory>* result) { |
||||
result->reset(new MockEnvDirectory()); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
bool MockEnv::FileExists(const std::string& fname) { |
||||
auto fn = NormalizePath(fname); |
||||
MutexLock lock(&mutex_); |
||||
if (file_map_.find(fn) != file_map_.end()) { |
||||
// File exists
|
||||
return true; |
||||
} |
||||
// Now also check if fn exists as a dir
|
||||
for (const auto& iter : file_map_) { |
||||
const std::string& filename = iter.first; |
||||
if (filename.size() >= fn.size() + 1 && |
||||
filename[fn.size()] == '/' && |
||||
Slice(filename).starts_with(Slice(fn))) { |
||||
return true; |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
Status MockEnv::GetChildren(const std::string& dir, |
||||
std::vector<std::string>* result) { |
||||
auto d = NormalizePath(dir); |
||||
{ |
||||
MutexLock lock(&mutex_); |
||||
result->clear(); |
||||
for (const auto& iter : file_map_) { |
||||
const std::string& filename = iter.first; |
||||
|
||||
if (filename.size() >= d.size() + 1 && filename[d.size()] == '/' && |
||||
Slice(filename).starts_with(Slice(d))) { |
||||
size_t next_slash = filename.find('/', d.size() + 1); |
||||
if (next_slash != std::string::npos) { |
||||
result->push_back(filename.substr( |
||||
d.size() + 1, next_slash - d.size() - 1)); |
||||
} else { |
||||
result->push_back(filename.substr(d.size() + 1)); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
result->erase(std::unique(result->begin(), result->end()), result->end()); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
void MockEnv::DeleteFileInternal(const std::string& fname) { |
||||
assert(fname == NormalizePath(fname)); |
||||
if (file_map_.find(fname) == file_map_.end()) { |
||||
return; |
||||
} |
||||
|
||||
file_map_[fname]->Unref(); |
||||
file_map_.erase(fname); |
||||
} |
||||
|
||||
Status MockEnv::DeleteFile(const std::string& fname) { |
||||
auto fn = NormalizePath(fname); |
||||
MutexLock lock(&mutex_); |
||||
if (file_map_.find(fn) == file_map_.end()) { |
||||
return Status::IOError(fn, "File not found"); |
||||
} |
||||
|
||||
DeleteFileInternal(fn); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::CreateDir(const std::string& dirname) { |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::CreateDirIfMissing(const std::string& dirname) { |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::DeleteDir(const std::string& dirname) { |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::GetFileSize(const std::string& fname, uint64_t* file_size) { |
||||
auto fn = NormalizePath(fname); |
||||
MutexLock lock(&mutex_); |
||||
auto iter = file_map_.find(fn); |
||||
if (iter == file_map_.end()) { |
||||
return Status::IOError(fn, "File not found"); |
||||
} |
||||
|
||||
*file_size = iter->second->Size(); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::GetFileModificationTime(const std::string& fname, |
||||
uint64_t* time) { |
||||
auto fn = NormalizePath(fname); |
||||
MutexLock lock(&mutex_); |
||||
auto iter = file_map_.find(fn); |
||||
if (iter == file_map_.end()) { |
||||
return Status::IOError(fn, "File not found"); |
||||
} |
||||
*time = iter->second->ModifiedTime(); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::RenameFile(const std::string& src, |
||||
const std::string& target) { |
||||
auto s = NormalizePath(src); |
||||
auto t = NormalizePath(target); |
||||
MutexLock lock(&mutex_); |
||||
if (file_map_.find(s) == file_map_.end()) { |
||||
return Status::IOError(s, "File not found"); |
||||
} |
||||
|
||||
DeleteFileInternal(t); |
||||
file_map_[t] = file_map_[s]; |
||||
file_map_.erase(s); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::NewLogger(const std::string& fname, |
||||
shared_ptr<Logger>* result) { |
||||
auto fn = NormalizePath(fname); |
||||
MutexLock lock(&mutex_); |
||||
auto iter = file_map_.find(fn); |
||||
MemFile* file = nullptr; |
||||
if (iter == file_map_.end()) { |
||||
file = new MemFile(fn); |
||||
file->Ref(); |
||||
file_map_[fn] = file; |
||||
} else { |
||||
file = iter->second; |
||||
} |
||||
std::unique_ptr<WritableFile> f(new WritableFileImpl(file, nullptr)); |
||||
result->reset(new TestMemLogger(std::move(f), this)); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::LockFile(const std::string& fname, FileLock** flock) { |
||||
auto fn = NormalizePath(fname); |
||||
{ |
||||
MutexLock lock(&mutex_); |
||||
if (file_map_.find(fn) != file_map_.end()) { |
||||
return Status::IOError(fn, "Lock file exists"); |
||||
} |
||||
file_map_[fn] = nullptr; |
||||
} |
||||
*flock = new MockEnvFileLock(fn); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::UnlockFile(FileLock* flock) { |
||||
std::string fn = dynamic_cast<MockEnvFileLock*>(flock)->FileName(); |
||||
{ |
||||
MutexLock lock(&mutex_); |
||||
auto iter = file_map_.find(fn); |
||||
if (iter != file_map_.end()) { |
||||
file_map_.erase(fn); |
||||
} |
||||
} |
||||
delete flock; |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status MockEnv::GetTestDirectory(std::string* path) { |
||||
*path = "/test"; |
||||
return Status::OK(); |
||||
} |
||||
|
||||
// Non-virtual functions, specific to MockEnv
|
||||
Status MockEnv::Truncate(const std::string& fname, size_t size) { |
||||
auto fn = NormalizePath(fname); |
||||
MutexLock lock(&mutex_); |
||||
auto iter = file_map_.find(fn); |
||||
if (iter == file_map_.end()) { |
||||
return Status::IOError(fn, "File not found"); |
||||
} |
||||
iter->second->Truncate(size); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
std::string MockEnv::NormalizePath(const std::string path) { |
||||
std::string dst; |
||||
for (auto c : path) { |
||||
if (!dst.empty() && c == '/' && dst.back() == '/') { |
||||
continue; |
||||
} |
||||
dst.push_back(c); |
||||
} |
||||
return dst; |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,93 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
#pragma once |
||||
|
||||
#include <atomic> |
||||
#include <map> |
||||
#include <string> |
||||
#include <vector> |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/status.h" |
||||
#include "port/port.h" |
||||
#include "util/mutexlock.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class MemFile; |
||||
class MockEnv : public EnvWrapper { |
||||
public: |
||||
explicit MockEnv(Env* base_env); |
||||
|
||||
virtual ~MockEnv(); |
||||
|
||||
// Partial implementation of the Env interface.
|
||||
virtual Status NewSequentialFile(const std::string& fname, |
||||
unique_ptr<SequentialFile>* result, |
||||
const EnvOptions& soptions); |
||||
|
||||
virtual Status NewRandomAccessFile(const std::string& fname, |
||||
unique_ptr<RandomAccessFile>* result, |
||||
const EnvOptions& soptions); |
||||
|
||||
virtual Status NewWritableFile(const std::string& fname, |
||||
unique_ptr<WritableFile>* result, |
||||
const EnvOptions& env_options); |
||||
|
||||
virtual Status NewRandomRWFile(const std::string& fname, |
||||
unique_ptr<RandomRWFile>* result, |
||||
const EnvOptions& options); |
||||
|
||||
virtual Status NewDirectory(const std::string& name, |
||||
unique_ptr<Directory>* result); |
||||
|
||||
virtual bool FileExists(const std::string& fname); |
||||
|
||||
virtual Status GetChildren(const std::string& dir, |
||||
std::vector<std::string>* result); |
||||
|
||||
void DeleteFileInternal(const std::string& fname); |
||||
|
||||
virtual Status DeleteFile(const std::string& fname); |
||||
|
||||
virtual Status CreateDir(const std::string& dirname); |
||||
|
||||
virtual Status CreateDirIfMissing(const std::string& dirname); |
||||
|
||||
virtual Status DeleteDir(const std::string& dirname); |
||||
|
||||
virtual Status GetFileSize(const std::string& fname, uint64_t* file_size); |
||||
|
||||
virtual Status GetFileModificationTime(const std::string& fname, |
||||
uint64_t* time); |
||||
|
||||
virtual Status RenameFile(const std::string& src, |
||||
const std::string& target); |
||||
|
||||
virtual Status NewLogger(const std::string& fname, |
||||
shared_ptr<Logger>* result); |
||||
|
||||
virtual Status LockFile(const std::string& fname, FileLock** flock); |
||||
|
||||
virtual Status UnlockFile(FileLock* flock); |
||||
|
||||
virtual Status GetTestDirectory(std::string* path); |
||||
|
||||
// Non-virtual functions, specific to MockEnv
|
||||
Status Truncate(const std::string& fname, size_t size); |
||||
|
||||
private: |
||||
std::string NormalizePath(const std::string path); |
||||
|
||||
// Map from filenames to MemFile objects, representing a simple file system.
|
||||
typedef std::map<std::string, MemFile*> FileSystem; |
||||
port::Mutex mutex_; |
||||
FileSystem file_map_; // Protected by mutex_.
|
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,232 @@ |
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <memory> |
||||
#include <string> |
||||
#include <vector> |
||||
#include "util/mock_env.h" |
||||
#include "db/db_impl.h" |
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "util/testharness.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class MockEnvTest { |
||||
public: |
||||
Env* env_; |
||||
const EnvOptions soptions_; |
||||
|
||||
MockEnvTest() |
||||
: env_(new MockEnv(Env::Default())) { |
||||
} |
||||
~MockEnvTest() { |
||||
delete env_; |
||||
} |
||||
}; |
||||
|
||||
TEST(MockEnvTest, Basics) { |
||||
uint64_t file_size; |
||||
unique_ptr<WritableFile> writable_file; |
||||
std::vector<std::string> children; |
||||
|
||||
ASSERT_OK(env_->CreateDir("/dir")); |
||||
|
||||
// Check that the directory is empty.
|
||||
ASSERT_TRUE(!env_->FileExists("/dir/non_existent")); |
||||
ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok()); |
||||
ASSERT_OK(env_->GetChildren("/dir", &children)); |
||||
ASSERT_EQ(0U, children.size()); |
||||
|
||||
// Create a file.
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_)); |
||||
writable_file.reset(); |
||||
|
||||
// Check that the file exists.
|
||||
ASSERT_TRUE(env_->FileExists("/dir/f")); |
||||
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); |
||||
ASSERT_EQ(0U, file_size); |
||||
ASSERT_OK(env_->GetChildren("/dir", &children)); |
||||
ASSERT_EQ(1U, children.size()); |
||||
ASSERT_EQ("f", children[0]); |
||||
|
||||
// Write to the file.
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_)); |
||||
ASSERT_OK(writable_file->Append("abc")); |
||||
writable_file.reset(); |
||||
|
||||
// Check for expected size.
|
||||
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); |
||||
ASSERT_EQ(3U, file_size); |
||||
|
||||
// Check that renaming works.
|
||||
ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok()); |
||||
ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g")); |
||||
ASSERT_TRUE(!env_->FileExists("/dir/f")); |
||||
ASSERT_TRUE(env_->FileExists("/dir/g")); |
||||
ASSERT_OK(env_->GetFileSize("/dir/g", &file_size)); |
||||
ASSERT_EQ(3U, file_size); |
||||
|
||||
// Check that opening non-existent file fails.
|
||||
unique_ptr<SequentialFile> seq_file; |
||||
unique_ptr<RandomAccessFile> rand_file; |
||||
ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file, |
||||
soptions_).ok()); |
||||
ASSERT_TRUE(!seq_file); |
||||
ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file, |
||||
soptions_).ok()); |
||||
ASSERT_TRUE(!rand_file); |
||||
|
||||
// Check that deleting works.
|
||||
ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok()); |
||||
ASSERT_OK(env_->DeleteFile("/dir/g")); |
||||
ASSERT_TRUE(!env_->FileExists("/dir/g")); |
||||
ASSERT_OK(env_->GetChildren("/dir", &children)); |
||||
ASSERT_EQ(0U, children.size()); |
||||
ASSERT_OK(env_->DeleteDir("/dir")); |
||||
} |
||||
|
||||
TEST(MockEnvTest, ReadWrite) { |
||||
unique_ptr<WritableFile> writable_file; |
||||
unique_ptr<SequentialFile> seq_file; |
||||
unique_ptr<RandomAccessFile> rand_file; |
||||
Slice result; |
||||
char scratch[100]; |
||||
|
||||
ASSERT_OK(env_->CreateDir("/dir")); |
||||
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_)); |
||||
ASSERT_OK(writable_file->Append("hello ")); |
||||
ASSERT_OK(writable_file->Append("world")); |
||||
writable_file.reset(); |
||||
|
||||
// Read sequentially.
|
||||
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file, soptions_)); |
||||
ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
|
||||
ASSERT_EQ(0, result.compare("hello")); |
||||
ASSERT_OK(seq_file->Skip(1)); |
||||
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world".
|
||||
ASSERT_EQ(0, result.compare("world")); |
||||
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF.
|
||||
ASSERT_EQ(0U, result.size()); |
||||
ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file.
|
||||
ASSERT_OK(seq_file->Read(1000, &result, scratch)); |
||||
ASSERT_EQ(0U, result.size()); |
||||
|
||||
// Random reads.
|
||||
ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file, soptions_)); |
||||
ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
|
||||
ASSERT_EQ(0, result.compare("world")); |
||||
ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
|
||||
ASSERT_EQ(0, result.compare("hello")); |
||||
ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d".
|
||||
ASSERT_EQ(0, result.compare("d")); |
||||
|
||||
// Too high offset.
|
||||
ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok()); |
||||
} |
||||
|
||||
TEST(MockEnvTest, Locks) { |
||||
FileLock* lock; |
||||
|
||||
// These are no-ops, but we test they return success.
|
||||
ASSERT_OK(env_->LockFile("some file", &lock)); |
||||
ASSERT_OK(env_->UnlockFile(lock)); |
||||
} |
||||
|
||||
TEST(MockEnvTest, Misc) { |
||||
std::string test_dir; |
||||
ASSERT_OK(env_->GetTestDirectory(&test_dir)); |
||||
ASSERT_TRUE(!test_dir.empty()); |
||||
|
||||
unique_ptr<WritableFile> writable_file; |
||||
ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file, soptions_)); |
||||
|
||||
// These are no-ops, but we test they return success.
|
||||
ASSERT_OK(writable_file->Sync()); |
||||
ASSERT_OK(writable_file->Flush()); |
||||
ASSERT_OK(writable_file->Close()); |
||||
writable_file.reset(); |
||||
} |
||||
|
||||
TEST(MockEnvTest, LargeWrite) { |
||||
const size_t kWriteSize = 300 * 1024; |
||||
char* scratch = new char[kWriteSize * 2]; |
||||
|
||||
std::string write_data; |
||||
for (size_t i = 0; i < kWriteSize; ++i) { |
||||
write_data.append(1, static_cast<char>(i)); |
||||
} |
||||
|
||||
unique_ptr<WritableFile> writable_file; |
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_)); |
||||
ASSERT_OK(writable_file->Append("foo")); |
||||
ASSERT_OK(writable_file->Append(write_data)); |
||||
writable_file.reset(); |
||||
|
||||
unique_ptr<SequentialFile> seq_file; |
||||
Slice result; |
||||
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file, soptions_)); |
||||
ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
|
||||
ASSERT_EQ(0, result.compare("foo")); |
||||
|
||||
size_t read = 0; |
||||
std::string read_data; |
||||
while (read < kWriteSize) { |
||||
ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch)); |
||||
read_data.append(result.data(), result.size()); |
||||
read += result.size(); |
||||
} |
||||
ASSERT_TRUE(write_data == read_data); |
||||
delete [] scratch; |
||||
} |
||||
|
||||
TEST(MockEnvTest, DBTest) { |
||||
Options options; |
||||
options.create_if_missing = true; |
||||
options.env = env_; |
||||
DB* db; |
||||
|
||||
const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")}; |
||||
const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")}; |
||||
|
||||
ASSERT_OK(DB::Open(options, "/dir/db", &db)); |
||||
for (size_t i = 0; i < 3; ++i) { |
||||
ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i])); |
||||
} |
||||
|
||||
for (size_t i = 0; i < 3; ++i) { |
||||
std::string res; |
||||
ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); |
||||
ASSERT_TRUE(res == vals[i]); |
||||
} |
||||
|
||||
Iterator* iterator = db->NewIterator(ReadOptions()); |
||||
iterator->SeekToFirst(); |
||||
for (size_t i = 0; i < 3; ++i) { |
||||
ASSERT_TRUE(iterator->Valid()); |
||||
ASSERT_TRUE(keys[i] == iterator->key()); |
||||
ASSERT_TRUE(vals[i] == iterator->value()); |
||||
iterator->Next(); |
||||
} |
||||
ASSERT_TRUE(!iterator->Valid()); |
||||
delete iterator; |
||||
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db); |
||||
ASSERT_OK(dbi->TEST_FlushMemTable()); |
||||
|
||||
for (size_t i = 0; i < 3; ++i) { |
||||
std::string res; |
||||
ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); |
||||
ASSERT_TRUE(res == vals[i]); |
||||
} |
||||
|
||||
delete db; |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
return rocksdb::test::RunAllTests(); |
||||
} |
||||
Loading…
Reference in new issue