NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Cleanup] Remove RandomRWFile

Browse Source 
Summary: RandomRWFile is not used anywhere in out code base, this patch remove RandomRWFile

Test Plan:
make check -j64
USE_CLANG=1 make all -j64
OPT=-DROCKSDB_LITE make release -j64

Reviewers: sdong, yhchiang, anthony, kradhakrishnan, rven, igor

Reviewed By: igor

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D44091
main
Islam AbdelRahman 10 years ago
parent c3466eab07
commit 3bd9db420e
12 changed files with 13 additions and 470 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 5
      HISTORY.md
  2. 10
      hdfs/env_hdfs.h
  3. 63
      include/rocksdb/env.h
  4. 144
      port/win/env_win.cc
  5. 6
      util/env_hdfs.cc
  6. 130
      util/env_posix.cc
  7. 22
      util/env_test.cc
  8. 53
      util/file_reader_writer.cc
  9. 20
      util/file_reader_writer.h
  10. 6
      util/mock_env.cc
  11. 4
      util/mock_env.h
  12. 20
      utilities/backupable/backupable_db_test.cc

5
HISTORY.md
Unescape View File

@ -1,5 +1,10 @@
# Rocksdb Change Log # Rocksdb Change Log
## Unreleased
### Public API Changes
* Removed class Env::RandomRWFile and Env::NewRandomRWFile().
## 3.13.0 (8/6/2015) ## 3.13.0 (8/6/2015)
### New Features ### New Features
* RollbackToSavePoint() in WriteBatch/WriteBatchWithIndex * RollbackToSavePoint() in WriteBatch/WriteBatchWithIndex

10
hdfs/env_hdfs.h
Unescape View File

@ -66,10 +66,6 @@ class HdfsEnv : public Env {
std::unique_ptr<WritableFile>* result, std::unique_ptr<WritableFile>* result,
const EnvOptions& options); const EnvOptions& options);
virtual Status NewRandomRWFile(const std::string& fname,
std::unique_ptr<RandomRWFile>* result,
const EnvOptions& options);
virtual Status NewDirectory(const std::string& name, virtual Status NewDirectory(const std::string& name,
std::unique_ptr<Directory>* result); std::unique_ptr<Directory>* result);
@ -268,12 +264,6 @@ class HdfsEnv : public Env {
return notsup; return notsup;
} }
virtual Status NewRandomRWFile(const std::string& fname,
unique_ptr<RandomRWFile>* result,
const EnvOptions& options) override {
return notsup;
}
virtual Status NewDirectory(const std::string& name, virtual Status NewDirectory(const std::string& name,
unique_ptr<Directory>* result) override { unique_ptr<Directory>* result) override {
return notsup; return notsup;

63
include/rocksdb/env.h
Unescape View File

@ -40,7 +40,6 @@ class RandomAccessFile;
class SequentialFile; class SequentialFile;
class Slice; class Slice;
class WritableFile; class WritableFile;
class RandomRWFile;
class Directory; class Directory;
struct DBOptions; struct DBOptions;
class RateLimiter; class RateLimiter;
@ -137,15 +136,6 @@ class Env {
unique_ptr<WritableFile>* result, unique_ptr<WritableFile>* result,
const EnvOptions& options) = 0; const EnvOptions& options) = 0;
// Create an object that both reads and writes to a file on
// specified offsets (random access). If file already exists,
// does not overwrite it. On success, stores a pointer to the
// new file in *result and returns OK. On failure stores nullptr
// in *result and returns non-OK.
virtual Status NewRandomRWFile(const std::string& fname,
unique_ptr<RandomRWFile>* result,
const EnvOptions& options) = 0;
// Create an object that represents a directory. Will fail if directory // Create an object that represents a directory. Will fail if directory
// doesn't exist. If the directory exists, it will open the directory // doesn't exist. If the directory exists, it will open the directory
// and create a new Directory object. // and create a new Directory object.
@ -568,55 +558,6 @@ class WritableFile {
Env::IOPriority io_priority_; Env::IOPriority io_priority_;
}; };
// A file abstraction for random reading and writing.
class RandomRWFile {
public:
RandomRWFile() {}
virtual ~RandomRWFile() {}
// Write data from Slice data to file starting from offset
// Returns IOError on failure, but does not guarantee
// atomicity of a write. Returns OK status on success.
//
// Safe for concurrent use.
virtual Status Write(uint64_t offset, const Slice& data) = 0;
// Read up to "n" bytes from the file starting at "offset".
// "scratch[0..n-1]" may be written by this routine. Sets "*result"
// to the data that was read (including if fewer than "n" bytes were
// successfully read). May set "*result" to point at data in
// "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
// "*result" is used. If an error was encountered, returns a non-OK
// status.
//
// Safe for concurrent use by multiple threads.
virtual Status Read(uint64_t offset, size_t n, Slice* result,
char* scratch) const = 0;
virtual Status Close() = 0; // closes the file
virtual Status Sync() = 0; // sync data
/*
* Sync data and/or metadata as well.
* By default, sync only data.
* Override this method for environments where we need to sync
* metadata as well.
*/
virtual Status Fsync() {
return Sync();
}
/*
* Pre-allocate space for a file.
*/
virtual Status Allocate(off_t offset, off_t len) {
return Status::OK();
}
private:
// No copying allowed
RandomRWFile(const RandomRWFile&);
void operator=(const RandomRWFile&);
};
// Directory object represents collection of files and implements // Directory object represents collection of files and implements
// filesystem operations that can be executed on directories. // filesystem operations that can be executed on directories.
class Directory { class Directory {
@ -766,10 +707,6 @@ class EnvWrapper : public Env {
const EnvOptions& options) override { const EnvOptions& options) override {
return target_->NewWritableFile(f, r, options); return target_->NewWritableFile(f, r, options);
} }
Status NewRandomRWFile(const std::string& f, unique_ptr<RandomRWFile>* r,
const EnvOptions& options) override {
return target_->NewRandomRWFile(f, r, options);
}
virtual Status NewDirectory(const std::string& name, virtual Status NewDirectory(const std::string& name,
unique_ptr<Directory>* result) override { unique_ptr<Directory>* result) override {
return target_->NewDirectory(name, result); return target_->NewDirectory(name, result);

144
port/win/env_win.cc
Unescape View File

@ -207,118 +207,6 @@ inline Status ftruncate(const std::string& filename, HANDLE hFile,
return status; return status;
} }
class WinRandomRWFile : public RandomRWFile {
const std::string filename_;
HANDLE hFile_;
bool pending_fsync_;
public:
WinRandomRWFile(const std::string& fname, HANDLE hFile,
const EnvOptions& options)
: filename_(fname), hFile_(hFile), pending_fsync_(false) {
assert(!options.use_mmap_writes && !options.use_mmap_reads);
}
~WinRandomRWFile() {
if (hFile_ != INVALID_HANDLE_VALUE && hFile_ != NULL) {
::CloseHandle(hFile_);
}
}
virtual Status Write(uint64_t offset, const Slice& data) override {
const char* src = data.data();
size_t left = data.size();
pending_fsync_ = true;
SSIZE_T done = 0;
{
IOSTATS_TIMER_GUARD(write_nanos);
done = pwrite(hFile_, src, left, offset);
}
if (done < 0) {
return IOErrorFromWindowsError("pwrite failed to: " + filename_,
GetLastError());
}
IOSTATS_ADD(bytes_written, done);
left -= done;
src += done;
offset += done;
return Status::OK();
}
virtual Status Read(uint64_t offset, size_t n, Slice* result,
char* scratch) const override {
Status s;
SSIZE_T r = -1;
char* ptr = scratch;
size_t left = n;
while (left > 0) {
{
IOSTATS_TIMER_GUARD(read_nanos);
r = pread(hFile_, ptr, n, offset);
}
if (r <= 0) {
break;
}
ptr += r;
offset += r;
left -= r;
}
IOSTATS_ADD_IF_POSITIVE(bytes_read, n - left);
*result = Slice(scratch, (r < 0) ? 0 : r);
if (r < 0) {
s = IOErrorFromWindowsError("pread failed from: " + filename_,
GetLastError());
}
return s;
}
virtual Status Close() override {
Status s = Status::OK();
if (hFile_ != INVALID_HANDLE_VALUE && ::CloseHandle(hFile_) == FALSE) {
s = IOErrorFromWindowsError("Failed to close file: " + filename_,
GetLastError());
}
hFile_ = INVALID_HANDLE_VALUE;
return s;
}
virtual Status Sync() override {
if (pending_fsync_ && fsync(hFile_) < 0) {
return IOErrorFromWindowsError(
"Failed to Sync() buffers for: " + filename_, GetLastError());
}
pending_fsync_ = false;
return Status::OK();
}
virtual Status Fsync() override {
if (pending_fsync_ && fsync(hFile_) < 0) {
return IOErrorFromWindowsError("Failed to Fsync() for: " + filename_,
GetLastError());
}
pending_fsync_ = false;
return Status::OK();
}
virtual Status Allocate(off_t offset, off_t len) override {
IOSTATS_TIMER_GUARD(allocate_nanos);
return fallocate(filename_, hFile_, len);
}
};
// mmap() based random-access // mmap() based random-access
class WinMmapReadableFile : public RandomAccessFile { class WinMmapReadableFile : public RandomAccessFile {
const std::string fileName_; const std::string fileName_;
@ -1616,38 +1504,6 @@ class WinEnv : public Env {
return s; return s;
} }
virtual Status NewRandomRWFile(const std::string& fname,
std::unique_ptr<RandomRWFile>* result,
const EnvOptions& options) override {
result->reset();
// no support for mmap yet (same as POSIX env)
if (options.use_mmap_writes || options.use_mmap_reads) {
return Status::NotSupported("No support for mmap read/write yet");
}
Status s;
HANDLE hFile = 0;
{
IOSTATS_TIMER_GUARD(open_nanos);
hFile = CreateFileA(fname.c_str(), GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ, NULL,
OPEN_ALWAYS, // Posix env specifies O_CREAT, it will
// open existing file or create new
FILE_ATTRIBUTE_NORMAL, NULL);
}
if (INVALID_HANDLE_VALUE == hFile) {
auto lastError = GetLastError();
s = IOErrorFromWindowsError(
"Failed to Open/Create NewRandomRWFile" + fname, lastError);
} else {
result->reset(new WinRandomRWFile(fname, hFile, options));
}
return s;
}
virtual Status NewDirectory(const std::string& name, virtual Status NewDirectory(const std::string& name,
std::unique_ptr<Directory>* result) override { std::unique_ptr<Directory>* result) override {
Status s; Status s;

6
util/env_hdfs.cc
Unescape View File

@ -415,12 +415,6 @@ Status HdfsEnv::NewWritableFile(const std::string& fname,
return Status::OK(); return Status::OK();
} }
Status HdfsEnv::NewRandomRWFile(const std::string& fname,
unique_ptr<RandomRWFile>* result,
const EnvOptions& options) {
return Status::NotSupported("NewRandomRWFile not supported on HdfsEnv");
}
class HdfsDirectory : public Directory { class HdfsDirectory : public Directory {
public: public:
explicit HdfsDirectory(int fd) : fd_(fd) {} explicit HdfsDirectory(int fd) : fd_(fd) {}

130
util/env_posix.cc
Unescape View File

@ -725,113 +725,6 @@ class PosixWritableFile : public WritableFile {
#endif #endif
}; };
class PosixRandomRWFile : public RandomRWFile {
private:
const std::string filename_;
int fd_;
#ifdef ROCKSDB_FALLOCATE_PRESENT
bool fallocate_with_keep_size_;
#endif
public:
PosixRandomRWFile(const std::string& fname, int fd, const EnvOptions& options)
: filename_(fname), fd_(fd) {
#ifdef ROCKSDB_FALLOCATE_PRESENT
fallocate_with_keep_size_ = options.fallocate_with_keep_size;
#endif
assert(!options.use_mmap_writes && !options.use_mmap_reads);
}
~PosixRandomRWFile() {
if (fd_ >= 0) {
Close();
}
}
virtual Status Write(uint64_t offset, const Slice& data) override {
const char* src = data.data();
size_t left = data.size();
Status s;
while (left != 0) {
ssize_t done = pwrite(fd_, src, left, offset);
if (done < 0) {
if (errno == EINTR) {
continue;
}
return IOError(filename_, errno);
}
left -= done;
src += done;
offset += done;
}
return Status::OK();
}
virtual Status Read(uint64_t offset, size_t n, Slice* result,
char* scratch) const override {
Status s;
ssize_t r = -1;
size_t left = n;
char* ptr = scratch;
while (left > 0) {
r = pread(fd_, ptr, left, static_cast<off_t>(offset));
if (r <= 0) {
if (errno == EINTR) {
continue;
}
break;
}
ptr += r;
offset += r;
left -= r;
}
*result = Slice(scratch, (r < 0) ? 0 : n - left);
if (r < 0) {
s = IOError(filename_, errno);
}
return s;
}
virtual Status Close() override {
Status s = Status::OK();
if (fd_ >= 0 && close(fd_) < 0) {
s = IOError(filename_, errno);
}
fd_ = -1;
return s;
}
virtual Status Sync() override {
if (fdatasync(fd_) < 0) {
return IOError(filename_, errno);
}
return Status::OK();
}
virtual Status Fsync() override {
if (fsync(fd_) < 0) {
return IOError(filename_, errno);
}
return Status::OK();
}
#ifdef ROCKSDB_FALLOCATE_PRESENT
virtual Status Allocate(off_t offset, off_t len) override {
IOSTATS_TIMER_GUARD(allocate_nanos);
int alloc_status = fallocate(
fd_, fallocate_with_keep_size_ ? FALLOC_FL_KEEP_SIZE : 0, offset, len);
if (alloc_status == 0) {
return Status::OK();
} else {
return IOError(filename_, errno);
}
}
#endif
};
class PosixDirectory : public Directory { class PosixDirectory : public Directory {
public: public:
explicit PosixDirectory(int fd) : fd_(fd) {} explicit PosixDirectory(int fd) : fd_(fd) {}
@ -1015,29 +908,6 @@ class PosixEnv : public Env {
return s; return s;
} }
virtual Status NewRandomRWFile(const std::string& fname,
unique_ptr<RandomRWFile>* result,
const EnvOptions& options) override {
result->reset();
// no support for mmap yet
if (options.use_mmap_writes || options.use_mmap_reads) {
return Status::NotSupported("No support for mmap read/write yet");
}
Status s;
int fd;
{
IOSTATS_TIMER_GUARD(open_nanos);
fd = open(fname.c_str(), O_CREAT | O_RDWR, 0644);
}
if (fd < 0) {
s = IOError(fname, errno);
} else {
SetFD_CLOEXEC(fd, &options);
result->reset(new PosixRandomRWFile(fname, fd, options));
}
return s;
}
virtual Status NewDirectory(const std::string& name, virtual Status NewDirectory(const std::string& name,
unique_ptr<Directory>* result) override { unique_ptr<Directory>* result) override {
result->reset(); result->reset();

22
util/env_test.cc
Unescape View File

@ -824,28 +824,6 @@ TEST_F(EnvPosixTest, InvalidateCache) {
#endif // not TRAVIS #endif // not TRAVIS
#endif // OS_LINUX #endif // OS_LINUX
TEST_F(EnvPosixTest, PosixRandomRWFileTest) {
EnvOptions soptions;
soptions.use_mmap_writes = soptions.use_mmap_reads = false;
std::string fname = test::TmpDir() + "/" + "testfile";
unique_ptr<RandomRWFile> file;
ASSERT_OK(env_->NewRandomRWFile(fname, &file, soptions));
// If you run the unit test on tmpfs, then tmpfs might not
// support fallocate. It is still better to trigger that
// code-path instead of eliminating it completely.
file.get()->Allocate(0, 10*1024*1024);
ASSERT_OK(file.get()->Write(100, Slice("Hello world")));
ASSERT_OK(file.get()->Write(105, Slice("Hello world")));
ASSERT_OK(file.get()->Sync());
ASSERT_OK(file.get()->Fsync());
char scratch[100];
Slice result;
ASSERT_OK(file.get()->Read(100, 16, &result, scratch));
ASSERT_EQ(result.compare("HelloHello world"), 0);
ASSERT_OK(file.get()->Close());
}
class TestLogger : public Logger { class TestLogger : public Logger {
public: public:
using Logger::Logv; using Logger::Logv;

53
util/file_reader_writer.cc
Unescape View File

@ -205,57 +205,4 @@ size_t WritableFileWriter::RequestToken(size_t bytes) {
} }
return bytes; return bytes;
} }
Status RandomRWFileAccessor::Write(uint64_t offset, const Slice& data) {
Status s;
pending_sync_ = true;
pending_fsync_ = true;
{
IOSTATS_TIMER_GUARD(write_nanos);
s = random_rw_file_->Write(offset, data);
if (!s.ok()) {
return s;
}
}
IOSTATS_ADD(bytes_written, data.size());
return s;
}
Status RandomRWFileAccessor::Read(uint64_t offset, size_t n, Slice* result,
char* scratch) const {
Status s;
{
IOSTATS_TIMER_GUARD(read_nanos);
s = random_rw_file_->Read(offset, n, result, scratch);
if (!s.ok()) {
return s;
}
}
IOSTATS_ADD_IF_POSITIVE(bytes_read, result->size());
return s;
}
Status RandomRWFileAccessor::Close() { return random_rw_file_->Close(); }
Status RandomRWFileAccessor::Sync(bool use_fsync) {
Status s;
if (pending_sync_) {
if (use_fsync) {
s = random_rw_file_->Fsync();
} else {
s = random_rw_file_->Sync();
}
if (!s.ok()) {
return s;
}
}
if (use_fsync) {
pending_fsync_ = false;
}
pending_sync_ = false;
return s;
}
} // namespace rocksdb } // namespace rocksdb

20
util/file_reader_writer.h
Unescape View File

@ -104,24 +104,4 @@ class WritableFileWriter {
size_t RequestToken(size_t bytes); size_t RequestToken(size_t bytes);
Status SyncInternal(bool use_fsync); Status SyncInternal(bool use_fsync);
}; };
class RandomRWFileAccessor {
private:
std::unique_ptr<RandomRWFile> random_rw_file_;
bool pending_sync_;
bool pending_fsync_;
public:
explicit RandomRWFileAccessor(std::unique_ptr<RandomRWFile>&& f)
: random_rw_file_(std::move(f)),
pending_sync_(false),
pending_fsync_(false) {}
Status Write(uint64_t offset, const Slice& data);
Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const;
Status Close();
Status Sync(bool use_fsync);
};
} // namespace rocksdb } // namespace rocksdb

6
util/mock_env.cc
Unescape View File

@ -456,12 +456,6 @@ Status MockEnv::NewWritableFile(const std::string& fname,
return Status::OK(); return Status::OK();
} }
Status MockEnv::NewRandomRWFile(const std::string& fname,
unique_ptr<RandomRWFile>* result,
const EnvOptions& options) {
return Status::OK();
}
Status MockEnv::NewDirectory(const std::string& name, Status MockEnv::NewDirectory(const std::string& name,
unique_ptr<Directory>* result) { unique_ptr<Directory>* result) {
result->reset(new MockEnvDirectory()); result->reset(new MockEnvDirectory());

4
util/mock_env.h
Unescape View File

@ -39,10 +39,6 @@ class MockEnv : public EnvWrapper {
unique_ptr<WritableFile>* result, unique_ptr<WritableFile>* result,
const EnvOptions& env_options) override; const EnvOptions& env_options) override;
virtual Status NewRandomRWFile(const std::string& fname,
unique_ptr<RandomRWFile>* result,
const EnvOptions& options) override;
virtual Status NewDirectory(const std::string& name, virtual Status NewDirectory(const std::string& name,
unique_ptr<Directory>* result) override; unique_ptr<Directory>* result) override;

20
utilities/backupable/backupable_db_test.cc
Unescape View File

@ -283,26 +283,22 @@ class FileManager : public EnvWrapper {
} }
Status CorruptFile(const std::string& fname, uint64_t bytes_to_corrupt) { Status CorruptFile(const std::string& fname, uint64_t bytes_to_corrupt) {
uint64_t size; std::string file_contents;
Status s = GetFileSize(fname, &size); Status s = ReadFileToString(this, fname, &file_contents);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
unique_ptr<RandomRWFile> file; s = DeleteFile(fname);
EnvOptions env_options;
env_options.use_mmap_writes = false;
s = NewRandomRWFile(fname, &file, env_options);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
RandomRWFileAccessor accessor(std::move(file));
for (uint64_t i = 0; s.ok() && i < bytes_to_corrupt; ++i) { for (uint64_t i = 0; i < bytes_to_corrupt; ++i) {
std::string tmp; std::string tmp;
// write one random byte to a random position test::RandomString(&rnd_, 1, &tmp);
s = accessor.Write(rnd_.Next() % size, file_contents[rnd_.Next() % file_contents.size()] = tmp[0];
test::RandomString(&rnd_, 1, &tmp));
} }
return s; return WriteToFile(fname, file_contents);
} }
Status CorruptChecksum(const std::string& fname, bool appear_valid) { Status CorruptChecksum(const std::string& fname, bool appear_valid) {

Write Preview
Loading…
Cancel
Save