NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make ChRootEnv, EncryptedEnv, and TimedEnv into FileSystems (#7968)

Browse Source 
Summary:
These classes were wraps of Env that provided only extensions to the FileSystem functionality.  Changed the classes to be FileSystems and the wraps to be of the CompositeEnvWrapper.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7968

Reviewed By: anand1976

Differential Revision: D26900253

Pulled By: mrambacher

fbshipit-source-id: 94001d8024a3c54a1c11adadca2bac66c3af2a77
main
mrambacher 4 years ago committed by Facebook GitHub Bot
parent 0304352882
commit 1a343bc393
4 changed files with 611 additions and 401 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 213
      env/env_chroot.cc
  2. 482
      env/env_encryption.cc
  3. 133
      include/rocksdb/env_encryption.h
  4. 162
      utilities/env_timed.cc

213
env/env_chroot.cc vendored
Unescape View File

@ -16,14 +16,17 @@
#include <utility> #include <utility>
#include <vector> #include <vector>
#include "env/composite_env_wrapper.h"
#include "rocksdb/file_system.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
namespace {
class ChrootEnv : public EnvWrapper { class ChrootFileSystem : public FileSystemWrapper {
public: public:
ChrootEnv(Env* base_env, const std::string& chroot_dir) ChrootFileSystem(const std::shared_ptr<FileSystem>& base,
: EnvWrapper(base_env) { const std::string& chroot_dir)
: FileSystemWrapper(base) {
#if defined(OS_AIX) #if defined(OS_AIX)
char resolvedName[PATH_MAX]; char resolvedName[PATH_MAX];
char* real_chroot_dir = realpath(chroot_dir.c_str(), resolvedName); char* real_chroot_dir = realpath(chroot_dir.c_str(), resolvedName);
@ -38,6 +41,7 @@ class ChrootEnv : public EnvWrapper {
#endif #endif
} }
const char* Name() const override { return "ChrootFS"; }
Status RegisterDbPaths(const std::vector<std::string>& paths) override { Status RegisterDbPaths(const std::vector<std::string>& paths) override {
std::vector<std::string> encoded_paths; std::vector<std::string> encoded_paths;
encoded_paths.reserve(paths.size()); encoded_paths.reserve(paths.size());
@ -48,7 +52,7 @@ class ChrootEnv : public EnvWrapper {
} }
encoded_paths.emplace_back(status_and_enc_path.second); encoded_paths.emplace_back(status_and_enc_path.second);
} }
return EnvWrapper::Env::RegisterDbPaths(encoded_paths); return FileSystemWrapper::RegisterDbPaths(encoded_paths);
} }
Status UnregisterDbPaths(const std::vector<std::string>& paths) override { Status UnregisterDbPaths(const std::vector<std::string>& paths) override {
@ -61,46 +65,49 @@ class ChrootEnv : public EnvWrapper {
} }
encoded_paths.emplace_back(status_and_enc_path.second); encoded_paths.emplace_back(status_and_enc_path.second);
} }
return EnvWrapper::Env::UnregisterDbPaths(encoded_paths); return FileSystemWrapper::UnregisterDbPaths(encoded_paths);
} }
Status NewSequentialFile(const std::string& fname, IOStatus NewSequentialFile(const std::string& fname,
std::unique_ptr<SequentialFile>* result, const FileOptions& options,
const EnvOptions& options) override { std::unique_ptr<FSSequentialFile>* result,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePathWithNewBasename(fname); auto status_and_enc_path = EncodePathWithNewBasename(fname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::NewSequentialFile(status_and_enc_path.second, result, return FileSystemWrapper::NewSequentialFile(status_and_enc_path.second,
options); options, result, dbg);
} }
Status NewRandomAccessFile(const std::string& fname, IOStatus NewRandomAccessFile(const std::string& fname,
std::unique_ptr<RandomAccessFile>* result, const FileOptions& options,
const EnvOptions& options) override { std::unique_ptr<FSRandomAccessFile>* result,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePathWithNewBasename(fname); auto status_and_enc_path = EncodePathWithNewBasename(fname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::NewRandomAccessFile(status_and_enc_path.second, result, return FileSystemWrapper::NewRandomAccessFile(status_and_enc_path.second,
options); options, result, dbg);
} }
Status NewWritableFile(const std::string& fname, IOStatus NewWritableFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<WritableFile>* result, std::unique_ptr<FSWritableFile>* result,
const EnvOptions& options) override { IODebugContext* dbg) override {
auto status_and_enc_path = EncodePathWithNewBasename(fname); auto status_and_enc_path = EncodePathWithNewBasename(fname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::NewWritableFile(status_and_enc_path.second, result, return FileSystemWrapper::NewWritableFile(status_and_enc_path.second,
options); options, result, dbg);
} }
Status ReuseWritableFile(const std::string& fname, IOStatus ReuseWritableFile(const std::string& fname,
const std::string& old_fname, const std::string& old_fname,
std::unique_ptr<WritableFile>* result, const FileOptions& options,
const EnvOptions& options) override { std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePathWithNewBasename(fname); auto status_and_enc_path = EncodePathWithNewBasename(fname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
@ -109,109 +116,131 @@ class ChrootEnv : public EnvWrapper {
if (!status_and_old_enc_path.first.ok()) { if (!status_and_old_enc_path.first.ok()) {
return status_and_old_enc_path.first; return status_and_old_enc_path.first;
} }
return EnvWrapper::ReuseWritableFile(status_and_old_enc_path.second, return FileSystemWrapper::ReuseWritableFile(status_and_old_enc_path.second,
status_and_old_enc_path.second, result, status_and_old_enc_path.second,
options); options, result, dbg);
} }
Status NewRandomRWFile(const std::string& fname, IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<RandomRWFile>* result, std::unique_ptr<FSRandomRWFile>* result,
const EnvOptions& options) override { IODebugContext* dbg) override {
auto status_and_enc_path = EncodePathWithNewBasename(fname); auto status_and_enc_path = EncodePathWithNewBasename(fname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::NewRandomRWFile(status_and_enc_path.second, result, return FileSystemWrapper::NewRandomRWFile(status_and_enc_path.second,
options); options, result, dbg);
} }
Status NewDirectory(const std::string& dir, IOStatus NewDirectory(const std::string& dir, const IOOptions& options,
std::unique_ptr<Directory>* result) override { std::unique_ptr<FSDirectory>* result,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePathWithNewBasename(dir); auto status_and_enc_path = EncodePathWithNewBasename(dir);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::NewDirectory(status_and_enc_path.second, result); return FileSystemWrapper::NewDirectory(status_and_enc_path.second, options,
result, dbg);
} }
Status FileExists(const std::string& fname) override { IOStatus FileExists(const std::string& fname, const IOOptions& options,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePathWithNewBasename(fname); auto status_and_enc_path = EncodePathWithNewBasename(fname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::FileExists(status_and_enc_path.second); return FileSystemWrapper::FileExists(status_and_enc_path.second, options,
dbg);
} }
Status GetChildren(const std::string& dir, IOStatus GetChildren(const std::string& dir, const IOOptions& options,
std::vector<std::string>* result) override { std::vector<std::string>* result,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePath(dir); auto status_and_enc_path = EncodePath(dir);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::GetChildren(status_and_enc_path.second, result); return FileSystemWrapper::GetChildren(status_and_enc_path.second, options,
result, dbg);
} }
Status GetChildrenFileAttributes( IOStatus GetChildrenFileAttributes(const std::string& dir,
const std::string& dir, std::vector<FileAttributes>* result) override { const IOOptions& options,
std::vector<FileAttributes>* result,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePath(dir); auto status_and_enc_path = EncodePath(dir);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::GetChildrenFileAttributes(status_and_enc_path.second, return FileSystemWrapper::GetChildrenFileAttributes(
result); status_and_enc_path.second, options, result, dbg);
} }
Status DeleteFile(const std::string& fname) override { IOStatus DeleteFile(const std::string& fname, const IOOptions& options,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePath(fname); auto status_and_enc_path = EncodePath(fname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::DeleteFile(status_and_enc_path.second); return FileSystemWrapper::DeleteFile(status_and_enc_path.second, options,
dbg);
} }
Status CreateDir(const std::string& dirname) override { IOStatus CreateDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePathWithNewBasename(dirname); auto status_and_enc_path = EncodePathWithNewBasename(dirname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::CreateDir(status_and_enc_path.second); return FileSystemWrapper::CreateDir(status_and_enc_path.second, options,
dbg);
} }
Status CreateDirIfMissing(const std::string& dirname) override { IOStatus CreateDirIfMissing(const std::string& dirname,
const IOOptions& options,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePathWithNewBasename(dirname); auto status_and_enc_path = EncodePathWithNewBasename(dirname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::CreateDirIfMissing(status_and_enc_path.second); return FileSystemWrapper::CreateDirIfMissing(status_and_enc_path.second,
options, dbg);
} }
Status DeleteDir(const std::string& dirname) override { IOStatus DeleteDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePath(dirname); auto status_and_enc_path = EncodePath(dirname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::DeleteDir(status_and_enc_path.second); return FileSystemWrapper::DeleteDir(status_and_enc_path.second, options,
dbg);
} }
Status GetFileSize(const std::string& fname, uint64_t* file_size) override { IOStatus GetFileSize(const std::string& fname, const IOOptions& options,
uint64_t* file_size, IODebugContext* dbg) override {
auto status_and_enc_path = EncodePath(fname); auto status_and_enc_path = EncodePath(fname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::GetFileSize(status_and_enc_path.second, file_size); return FileSystemWrapper::GetFileSize(status_and_enc_path.second, options,
file_size, dbg);
} }
Status GetFileModificationTime(const std::string& fname, IOStatus GetFileModificationTime(const std::string& fname,
uint64_t* file_mtime) override { const IOOptions& options,
uint64_t* file_mtime,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePath(fname); auto status_and_enc_path = EncodePath(fname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::GetFileModificationTime(status_and_enc_path.second, return FileSystemWrapper::GetFileModificationTime(
file_mtime); status_and_enc_path.second, options, file_mtime, dbg);
} }
Status RenameFile(const std::string& src, const std::string& dest) override { IOStatus RenameFile(const std::string& src, const std::string& dest,
const IOOptions& options, IODebugContext* dbg) override {
auto status_and_src_enc_path = EncodePath(src); auto status_and_src_enc_path = EncodePath(src);
if (!status_and_src_enc_path.first.ok()) { if (!status_and_src_enc_path.first.ok()) {
return status_and_src_enc_path.first; return status_and_src_enc_path.first;
@ -220,11 +249,13 @@ class ChrootEnv : public EnvWrapper {
if (!status_and_dest_enc_path.first.ok()) { if (!status_and_dest_enc_path.first.ok()) {
return status_and_dest_enc_path.first; return status_and_dest_enc_path.first;
} }
return EnvWrapper::RenameFile(status_and_src_enc_path.second, return FileSystemWrapper::RenameFile(status_and_src_enc_path.second,
status_and_dest_enc_path.second); status_and_dest_enc_path.second,
options, dbg);
} }
Status LinkFile(const std::string& src, const std::string& dest) override { IOStatus LinkFile(const std::string& src, const std::string& dest,
const IOOptions& options, IODebugContext* dbg) override {
auto status_and_src_enc_path = EncodePath(src); auto status_and_src_enc_path = EncodePath(src);
if (!status_and_src_enc_path.first.ok()) { if (!status_and_src_enc_path.first.ok()) {
return status_and_src_enc_path.first; return status_and_src_enc_path.first;
@ -233,11 +264,13 @@ class ChrootEnv : public EnvWrapper {
if (!status_and_dest_enc_path.first.ok()) { if (!status_and_dest_enc_path.first.ok()) {
return status_and_dest_enc_path.first; return status_and_dest_enc_path.first;
} }
return EnvWrapper::LinkFile(status_and_src_enc_path.second, return FileSystemWrapper::LinkFile(status_and_src_enc_path.second,
status_and_dest_enc_path.second); status_and_dest_enc_path.second, options,
dbg);
} }
Status LockFile(const std::string& fname, FileLock** lock) override { IOStatus LockFile(const std::string& fname, const IOOptions& options,
FileLock** lock, IODebugContext* dbg) override {
auto status_and_enc_path = EncodePathWithNewBasename(fname); auto status_and_enc_path = EncodePathWithNewBasename(fname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
@ -245,10 +278,12 @@ class ChrootEnv : public EnvWrapper {
// FileLock subclasses may store path (e.g., PosixFileLock stores it). We // FileLock subclasses may store path (e.g., PosixFileLock stores it). We
// can skip stripping the chroot directory from this path because callers // can skip stripping the chroot directory from this path because callers
// shouldn't use it. // shouldn't use it.
return EnvWrapper::LockFile(status_and_enc_path.second, lock); return FileSystemWrapper::LockFile(status_and_enc_path.second, options,
lock, dbg);
} }
Status GetTestDirectory(std::string* path) override { IOStatus GetTestDirectory(const IOOptions& options, std::string* path,
IODebugContext* dbg) override {
// Adapted from PosixEnv's implementation since it doesn't provide a way to // Adapted from PosixEnv's implementation since it doesn't provide a way to
// create directory in the chroot. // create directory in the chroot.
char buf[256]; char buf[256];
@ -256,36 +291,40 @@ class ChrootEnv : public EnvWrapper {
*path = buf; *path = buf;
// Directory may already exist, so ignore return // Directory may already exist, so ignore return
return CreateDirIfMissing(*path); return CreateDirIfMissing(*path, options, dbg);
} }
Status NewLogger(const std::string& fname, IOStatus NewLogger(const std::string& fname, const IOOptions& options,
std::shared_ptr<Logger>* result) override { std::shared_ptr<Logger>* result,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePathWithNewBasename(fname); auto status_and_enc_path = EncodePathWithNewBasename(fname);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::NewLogger(status_and_enc_path.second, result); return FileSystemWrapper::NewLogger(status_and_enc_path.second, options,
result, dbg);
} }
Status GetAbsolutePath(const std::string& db_path, IOStatus GetAbsolutePath(const std::string& db_path, const IOOptions& options,
std::string* output_path) override { std::string* output_path,
IODebugContext* dbg) override {
auto status_and_enc_path = EncodePath(db_path); auto status_and_enc_path = EncodePath(db_path);
if (!status_and_enc_path.first.ok()) { if (!status_and_enc_path.first.ok()) {
return status_and_enc_path.first; return status_and_enc_path.first;
} }
return EnvWrapper::GetAbsolutePath(status_and_enc_path.second, output_path); return FileSystemWrapper::GetAbsolutePath(status_and_enc_path.second,
options, output_path, dbg);
} }
private: private:
// Returns status and expanded absolute path including the chroot directory. // Returns status and expanded absolute path including the chroot directory.
// Checks whether the provided path breaks out of the chroot. If it returns // Checks whether the provided path breaks out of the chroot. If it returns
// non-OK status, the returned path should not be used. // non-OK status, the returned path should not be used.
std::pair<Status, std::string> EncodePath(const std::string& path) { std::pair<IOStatus, std::string> EncodePath(const std::string& path) {
if (path.empty() || path[0] != '/') { if (path.empty() || path[0] != '/') {
return {Status::InvalidArgument(path, "Not an absolute path"), ""}; return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""};
} }
std::pair<Status, std::string> res; std::pair<IOStatus, std::string> res;
res.second = chroot_dir_ + path; res.second = chroot_dir_ + path;
#if defined(OS_AIX) #if defined(OS_AIX)
char resolvedName[PATH_MAX]; char resolvedName[PATH_MAX];
@ -294,14 +333,14 @@ class ChrootEnv : public EnvWrapper {
char* normalized_path = realpath(res.second.c_str(), nullptr); char* normalized_path = realpath(res.second.c_str(), nullptr);
#endif #endif
if (normalized_path == nullptr) { if (normalized_path == nullptr) {
res.first = Status::NotFound(res.second, strerror(errno)); res.first = IOStatus::NotFound(res.second, strerror(errno));
} else if (strlen(normalized_path) < chroot_dir_.size() || } else if (strlen(normalized_path) < chroot_dir_.size() ||
strncmp(normalized_path, chroot_dir_.c_str(), strncmp(normalized_path, chroot_dir_.c_str(),
chroot_dir_.size()) != 0) { chroot_dir_.size()) != 0) {
res.first = Status::IOError(res.second, res.first = IOStatus::IOError(res.second,
"Attempted to access path outside chroot"); "Attempted to access path outside chroot");
} else { } else {
res.first = Status::OK(); res.first = IOStatus::OK();
} }
#if !defined(OS_AIX) #if !defined(OS_AIX)
free(normalized_path); free(normalized_path);
@ -311,10 +350,10 @@ class ChrootEnv : public EnvWrapper {
// Similar to EncodePath() except assumes the basename in the path hasn't been // Similar to EncodePath() except assumes the basename in the path hasn't been
// created yet. // created yet.
std::pair<Status, std::string> EncodePathWithNewBasename( std::pair<IOStatus, std::string> EncodePathWithNewBasename(
const std::string& path) { const std::string& path) {
if (path.empty() || path[0] != '/') { if (path.empty() || path[0] != '/') {
return {Status::InvalidArgument(path, "Not an absolute path"), ""}; return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""};
} }
// Basename may be followed by trailing slashes // Basename may be followed by trailing slashes
size_t final_idx = path.find_last_not_of('/'); size_t final_idx = path.find_last_not_of('/');
@ -333,12 +372,20 @@ class ChrootEnv : public EnvWrapper {
std::string chroot_dir_; std::string chroot_dir_;
}; };
} // namespace
std::shared_ptr<FileSystem> NewChrootFileSystem(
const std::shared_ptr<FileSystem>& base, const std::string& chroot_dir) {
return std::make_shared<ChrootFileSystem>(base, chroot_dir);
}
Env* NewChrootEnv(Env* base_env, const std::string& chroot_dir) { Env* NewChrootEnv(Env* base_env, const std::string& chroot_dir) {
if (!base_env->FileExists(chroot_dir).ok()) { if (!base_env->FileExists(chroot_dir).ok()) {
return nullptr; return nullptr;
} }
return new ChrootEnv(base_env, chroot_dir); std::shared_ptr<FileSystem> chroot_fs =
NewChrootFileSystem(base_env->GetFileSystem(), chroot_dir);
return new CompositeEnvWrapper(base_env, chroot_fs);
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

482
env/env_encryption.cc vendored
Unescape View File

@ -12,9 +12,11 @@
#include <cctype> #include <cctype>
#include <iostream> #include <iostream>
#include "env/composite_env_wrapper.h"
#include "env/env_encryption_ctr.h" #include "env/env_encryption_ctr.h"
#include "monitoring/perf_context_imp.h" #include "monitoring/perf_context_imp.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/io_status.h"
#include "rocksdb/system_clock.h" #include "rocksdb/system_clock.h"
#include "util/aligned_buffer.h" #include "util/aligned_buffer.h"
#include "util/coding.h" #include "util/coding.h"
@ -85,19 +87,24 @@ std::shared_ptr<EncryptionProvider> EncryptionProvider::NewCTRProvider(
// If an error was encountered, returns a non-OK status. // If an error was encountered, returns a non-OK status.
// //
// REQUIRES: External synchronization // REQUIRES: External synchronization
Status EncryptedSequentialFile::Read(size_t n, Slice* result, char* scratch) { IOStatus EncryptedSequentialFile::Read(size_t n, const IOOptions& options,
Slice* result, char* scratch,
IODebugContext* dbg) {
assert(scratch); assert(scratch);
Status status = file_->Read(n, result, scratch); IOStatus io_s = file_->Read(n, options, result, scratch, dbg);
if (!status.ok()) { if (!io_s.ok()) {
return status; return io_s;
} }
{ {
PERF_TIMER_GUARD(decrypt_data_nanos); PERF_TIMER_GUARD(decrypt_data_nanos);
status = stream_->Decrypt(offset_, (char*)result->data(), result->size()); io_s = status_to_io_status(
stream_->Decrypt(offset_, (char*)result->data(), result->size()));
} }
if (io_s.ok()) {
offset_ += result->size(); // We've already ready data from disk, so update offset_ += result->size(); // We've already ready data from disk, so update
// offset_ even if decryption fails. // offset_ even if decryption fails.
return status; }
return io_s;
} }
// Skip "n" bytes from the file. This is guaranteed to be no // Skip "n" bytes from the file. This is guaranteed to be no
@ -107,7 +114,7 @@ Status EncryptedSequentialFile::Read(size_t n, Slice* result, char* scratch) {
// file, and Skip will return OK. // file, and Skip will return OK.
// //
// REQUIRES: External synchronization // REQUIRES: External synchronization
Status EncryptedSequentialFile::Skip(uint64_t n) { IOStatus EncryptedSequentialFile::Skip(uint64_t n) {
auto status = file_->Skip(n); auto status = file_->Skip(n);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
@ -131,26 +138,30 @@ size_t EncryptedSequentialFile::GetRequiredBufferAlignment() const {
// Remove any kind of caching of data from the offset to offset+length // Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file. // of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop. // If the system is not caching the file contents, then this is a noop.
Status EncryptedSequentialFile::InvalidateCache(size_t offset, size_t length) { IOStatus EncryptedSequentialFile::InvalidateCache(size_t offset,
size_t length) {
return file_->InvalidateCache(offset + prefixLength_, length); return file_->InvalidateCache(offset + prefixLength_, length);
} }
// Positioned Read for direct I/O // Positioned Read for direct I/O
// If Direct I/O enabled, offset, n, and scratch should be properly aligned // If Direct I/O enabled, offset, n, and scratch should be properly aligned
Status EncryptedSequentialFile::PositionedRead(uint64_t offset, size_t n, IOStatus EncryptedSequentialFile::PositionedRead(uint64_t offset, size_t n,
Slice* result, char* scratch) { const IOOptions& options,
Slice* result, char* scratch,
IODebugContext* dbg) {
assert(scratch); assert(scratch);
offset += prefixLength_; // Skip prefix offset += prefixLength_; // Skip prefix
auto status = file_->PositionedRead(offset, n, result, scratch); auto io_s = file_->PositionedRead(offset, n, options, result, scratch, dbg);
if (!status.ok()) { if (!io_s.ok()) {
return status; return io_s;
} }
offset_ = offset + result->size(); offset_ = offset + result->size();
{ {
PERF_TIMER_GUARD(decrypt_data_nanos); PERF_TIMER_GUARD(decrypt_data_nanos);
status = stream_->Decrypt(offset, (char*)result->data(), result->size()); io_s = status_to_io_status(
stream_->Decrypt(offset, (char*)result->data(), result->size()));
} }
return status; return io_s;
} }
// Read up to "n" bytes from the file starting at "offset". // Read up to "n" bytes from the file starting at "offset".
@ -163,25 +174,30 @@ Status EncryptedSequentialFile::PositionedRead(uint64_t offset, size_t n,
// //
// Safe for concurrent use by multiple threads. // Safe for concurrent use by multiple threads.
// If Direct I/O enabled, offset, n, and scratch should be aligned properly. // If Direct I/O enabled, offset, n, and scratch should be aligned properly.
Status EncryptedRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result, IOStatus EncryptedRandomAccessFile::Read(uint64_t offset, size_t n,
char* scratch) const { const IOOptions& options,
Slice* result, char* scratch,
IODebugContext* dbg) const {
assert(scratch); assert(scratch);
offset += prefixLength_; offset += prefixLength_;
auto status = file_->Read(offset, n, result, scratch); auto io_s = file_->Read(offset, n, options, result, scratch, dbg);
if (!status.ok()) { if (!io_s.ok()) {
return status; return io_s;
} }
{ {
PERF_TIMER_GUARD(decrypt_data_nanos); PERF_TIMER_GUARD(decrypt_data_nanos);
status = stream_->Decrypt(offset, (char*)result->data(), result->size()); io_s = status_to_io_status(
stream_->Decrypt(offset, (char*)result->data(), result->size()));
} }
return status; return io_s;
} }
// Readahead the file starting from offset by n bytes for caching. // Readahead the file starting from offset by n bytes for caching.
Status EncryptedRandomAccessFile::Prefetch(uint64_t offset, size_t n) { IOStatus EncryptedRandomAccessFile::Prefetch(uint64_t offset, size_t n,
const IOOptions& options,
IODebugContext* dbg) {
// return Status::OK(); // return Status::OK();
return file_->Prefetch(offset + prefixLength_, n); return file_->Prefetch(offset + prefixLength_, n, options, dbg);
} }
// Tries to get an unique ID for this file that will be the same each time // Tries to get an unique ID for this file that will be the same each time
@ -222,7 +238,7 @@ size_t EncryptedRandomAccessFile::GetRequiredBufferAlignment() const {
// Remove any kind of caching of data from the offset to offset+length // Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file. // of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop. // If the system is not caching the file contents, then this is a noop.
Status EncryptedRandomAccessFile::InvalidateCache(size_t offset, IOStatus EncryptedRandomAccessFile::InvalidateCache(size_t offset,
size_t length) { size_t length) {
return file_->InvalidateCache(offset + prefixLength_, length); return file_->InvalidateCache(offset + prefixLength_, length);
} }
@ -230,12 +246,13 @@ Status EncryptedRandomAccessFile::InvalidateCache(size_t offset,
// A file abstraction for sequential writing. The implementation // A file abstraction for sequential writing. The implementation
// must provide buffering since callers may append small fragments // must provide buffering since callers may append small fragments
// at a time to the file. // at a time to the file.
Status EncryptedWritableFile::Append(const Slice& data) { IOStatus EncryptedWritableFile::Append(const Slice& data,
const IOOptions& options,
IODebugContext* dbg) {
AlignedBuffer buf; AlignedBuffer buf;
Status status;
Slice dataToAppend(data); Slice dataToAppend(data);
if (data.size() > 0) { if (data.size() > 0) {
auto offset = file_->GetFileSize(); // size including prefix auto offset = file_->GetFileSize(options, dbg); // size including prefix
// Encrypt in cloned buffer // Encrypt in cloned buffer
buf.Alignment(GetRequiredBufferAlignment()); buf.Alignment(GetRequiredBufferAlignment());
buf.AllocateNewBuffer(data.size()); buf.AllocateNewBuffer(data.size());
@ -243,26 +260,25 @@ Status EncryptedWritableFile::Append(const Slice& data) {
// so that the next two lines can be replaced with buf.Append(). // so that the next two lines can be replaced with buf.Append().
memmove(buf.BufferStart(), data.data(), data.size()); memmove(buf.BufferStart(), data.data(), data.size());
buf.Size(data.size()); buf.Size(data.size());
IOStatus io_s;
{ {
PERF_TIMER_GUARD(encrypt_data_nanos); PERF_TIMER_GUARD(encrypt_data_nanos);
status = stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize()); io_s = status_to_io_status(
stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize()));
} }
if (!status.ok()) { if (!io_s.ok()) {
return status; return io_s;
} }
dataToAppend = Slice(buf.BufferStart(), buf.CurrentSize()); dataToAppend = Slice(buf.BufferStart(), buf.CurrentSize());
} }
status = file_->Append(dataToAppend); return file_->Append(dataToAppend, options, dbg);
if (!status.ok()) {
return status;
}
return status;
} }
Status EncryptedWritableFile::PositionedAppend(const Slice& data, IOStatus EncryptedWritableFile::PositionedAppend(const Slice& data,
uint64_t offset) { uint64_t offset,
const IOOptions& options,
IODebugContext* dbg) {
AlignedBuffer buf; AlignedBuffer buf;
Status status;
Slice dataToAppend(data); Slice dataToAppend(data);
offset += prefixLength_; offset += prefixLength_;
if (data.size() > 0) { if (data.size() > 0) {
@ -271,20 +287,18 @@ Status EncryptedWritableFile::PositionedAppend(const Slice& data,
buf.AllocateNewBuffer(data.size()); buf.AllocateNewBuffer(data.size());
memmove(buf.BufferStart(), data.data(), data.size()); memmove(buf.BufferStart(), data.data(), data.size());
buf.Size(data.size()); buf.Size(data.size());
IOStatus io_s;
{ {
PERF_TIMER_GUARD(encrypt_data_nanos); PERF_TIMER_GUARD(encrypt_data_nanos);
status = stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize()); io_s = status_to_io_status(
stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize()));
} }
if (!status.ok()) { if (!io_s.ok()) {
return status; return io_s;
} }
dataToAppend = Slice(buf.BufferStart(), buf.CurrentSize()); dataToAppend = Slice(buf.BufferStart(), buf.CurrentSize());
} }
status = file_->PositionedAppend(dataToAppend, offset); return file_->PositionedAppend(dataToAppend, offset, options, dbg);
if (!status.ok()) {
return status;
}
return status;
} }
// Indicates the upper layers if the current WritableFile implementation // Indicates the upper layers if the current WritableFile implementation
@ -302,48 +316,72 @@ size_t EncryptedWritableFile::GetRequiredBufferAlignment() const {
/* /*
* Get the size of valid data in the file. * Get the size of valid data in the file.
*/ */
uint64_t EncryptedWritableFile::GetFileSize() { uint64_t EncryptedWritableFile::GetFileSize(const IOOptions& options,
return file_->GetFileSize() - prefixLength_; IODebugContext* dbg) {
return file_->GetFileSize(options, dbg) - prefixLength_;
} }
// Truncate is necessary to trim the file to the correct size // Truncate is necessary to trim the file to the correct size
// before closing. It is not always possible to keep track of the file // before closing. It is not always possible to keep track of the file
// size due to whole pages writes. The behavior is undefined if called // size due to whole pages writes. The behavior is undefined if called
// with other writes to follow. // with other writes to follow.
Status EncryptedWritableFile::Truncate(uint64_t size) { IOStatus EncryptedWritableFile::Truncate(uint64_t size,
return file_->Truncate(size + prefixLength_); const IOOptions& options,
IODebugContext* dbg) {
return file_->Truncate(size + prefixLength_, options, dbg);
} }
// Remove any kind of caching of data from the offset to offset+length // Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file. // of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop. // If the system is not caching the file contents, then this is a noop.
// This call has no effect on dirty pages in the cache. // This call has no effect on dirty pages in the cache.
Status EncryptedWritableFile::InvalidateCache(size_t offset, size_t length) { IOStatus EncryptedWritableFile::InvalidateCache(size_t offset, size_t length) {
return file_->InvalidateCache(offset + prefixLength_, length); return file_->InvalidateCache(offset + prefixLength_, length);
} }
// Sync a file range with disk. // Sync a file range with disk.
// offset is the starting byte of the file range to be synchronized. // offset is the starting byte of the file range to be synchronized.
// nbytes specifies the length of the range to be synchronized. // nbytes specifies the length of the range to be synchronized.
// This asks the OS to initiate flushing the cached data to disk, // This asks the OS to initiate flushing the cached data to disk,
// without waiting for completion. // without waiting for completion.
// Default implementation does nothing. // Default implementation does nothing.
Status EncryptedWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) { IOStatus EncryptedWritableFile::RangeSync(uint64_t offset, uint64_t nbytes,
return file_->RangeSync(offset + prefixLength_, nbytes); const IOOptions& options,
IODebugContext* dbg) {
return file_->RangeSync(offset + prefixLength_, nbytes, options, dbg);
} }
// PrepareWrite performs any necessary preparation for a write // PrepareWrite performs any necessary preparation for a write
// before the write actually occurs. This allows for pre-allocation // before the write actually occurs. This allows for pre-allocation
// of space on devices where it can result in less file // of space on devices where it can result in less file
// fragmentation and/or less waste from over-zealous filesystem // fragmentation and/or less waste from over-zealous filesystem
// pre-allocation. // pre-allocation.
void EncryptedWritableFile::PrepareWrite(size_t offset, size_t len) { void EncryptedWritableFile::PrepareWrite(size_t offset, size_t len,
file_->PrepareWrite(offset + prefixLength_, len); const IOOptions& options,
IODebugContext* dbg) {
file_->PrepareWrite(offset + prefixLength_, len, options, dbg);
} }
// Pre-allocates space for a file. // Pre-allocates space for a file.
Status EncryptedWritableFile::Allocate(uint64_t offset, uint64_t len) { IOStatus EncryptedWritableFile::Allocate(uint64_t offset, uint64_t len,
return file_->Allocate(offset + prefixLength_, len); const IOOptions& options,
IODebugContext* dbg) {
return file_->Allocate(offset + prefixLength_, len, options, dbg);
}
IOStatus EncryptedWritableFile::Flush(const IOOptions& options,
IODebugContext* dbg) {
return file_->Flush(options, dbg);
}
IOStatus EncryptedWritableFile::Sync(const IOOptions& options,
IODebugContext* dbg) {
return file_->Sync(options, dbg);
}
IOStatus EncryptedWritableFile::Close(const IOOptions& options,
IODebugContext* dbg) {
return file_->Close(options, dbg);
} }
// A file abstraction for random reading and writing. // A file abstraction for random reading and writing.
@ -362,9 +400,10 @@ size_t EncryptedRandomRWFile::GetRequiredBufferAlignment() const {
// Write bytes in `data` at offset `offset`, Returns Status::OK() on success. // Write bytes in `data` at offset `offset`, Returns Status::OK() on success.
// Pass aligned buffer when use_direct_io() returns true. // Pass aligned buffer when use_direct_io() returns true.
Status EncryptedRandomRWFile::Write(uint64_t offset, const Slice& data) { IOStatus EncryptedRandomRWFile::Write(uint64_t offset, const Slice& data,
const IOOptions& options,
IODebugContext* dbg) {
AlignedBuffer buf; AlignedBuffer buf;
Status status;
Slice dataToWrite(data); Slice dataToWrite(data);
offset += prefixLength_; offset += prefixLength_;
if (data.size() > 0) { if (data.size() > 0) {
@ -373,71 +412,89 @@ Status EncryptedRandomRWFile::Write(uint64_t offset, const Slice& data) {
buf.AllocateNewBuffer(data.size()); buf.AllocateNewBuffer(data.size());
memmove(buf.BufferStart(), data.data(), data.size()); memmove(buf.BufferStart(), data.data(), data.size());
buf.Size(data.size()); buf.Size(data.size());
IOStatus io_s;
{ {
PERF_TIMER_GUARD(encrypt_data_nanos); PERF_TIMER_GUARD(encrypt_data_nanos);
status = stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize()); io_s = status_to_io_status(
stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize()));
} }
if (!status.ok()) { if (!io_s.ok()) {
return status; return io_s;
} }
dataToWrite = Slice(buf.BufferStart(), buf.CurrentSize()); dataToWrite = Slice(buf.BufferStart(), buf.CurrentSize());
} }
status = file_->Write(offset, dataToWrite); return file_->Write(offset, dataToWrite, options, dbg);
return status;
} }
// Read up to `n` bytes starting from offset `offset` and store them in // Read up to `n` bytes starting from offset `offset` and store them in
// result, provided `scratch` size should be at least `n`. // result, provided `scratch` size should be at least `n`.
// Returns Status::OK() on success. // Returns Status::OK() on success.
Status EncryptedRandomRWFile::Read(uint64_t offset, size_t n, Slice* result, IOStatus EncryptedRandomRWFile::Read(uint64_t offset, size_t n,
char* scratch) const { const IOOptions& options, Slice* result,
char* scratch, IODebugContext* dbg) const {
assert(scratch); assert(scratch);
offset += prefixLength_; offset += prefixLength_;
auto status = file_->Read(offset, n, result, scratch); auto status = file_->Read(offset, n, options, result, scratch, dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
{ {
PERF_TIMER_GUARD(decrypt_data_nanos); PERF_TIMER_GUARD(decrypt_data_nanos);
status = stream_->Decrypt(offset, (char*)result->data(), result->size()); status = status_to_io_status(
stream_->Decrypt(offset, (char*)result->data(), result->size()));
} }
return status; return status;
} }
Status EncryptedRandomRWFile::Flush() { return file_->Flush(); } IOStatus EncryptedRandomRWFile::Flush(const IOOptions& options,
IODebugContext* dbg) {
return file_->Flush(options, dbg);
}
Status EncryptedRandomRWFile::Sync() { return file_->Sync(); } IOStatus EncryptedRandomRWFile::Sync(const IOOptions& options,
IODebugContext* dbg) {
return file_->Sync(options, dbg);
}
Status EncryptedRandomRWFile::Fsync() { return file_->Fsync(); } IOStatus EncryptedRandomRWFile::Fsync(const IOOptions& options,
IODebugContext* dbg) {
return file_->Fsync(options, dbg);
}
Status EncryptedRandomRWFile::Close() { return file_->Close(); } IOStatus EncryptedRandomRWFile::Close(const IOOptions& options,
IODebugContext* dbg) {
return file_->Close(options, dbg);
}
// EncryptedEnv implements an Env wrapper that adds encryption to files stored namespace {
// on disk. // EncryptedFileSystemImpl implements an FileSystemWrapper that adds encryption
class EncryptedEnvImpl : public EnvWrapper { // to files stored on disk.
class EncryptedFileSystemImpl : public EncryptedFileSystem {
public:
const char* Name() const override { return "EncryptedFS"; }
// Returns the raw encryption provider that should be used to write the input // Returns the raw encryption provider that should be used to write the input
// encrypted file. If there is no such provider, NotFound is returned. // encrypted file. If there is no such provider, NotFound is returned.
Status GetWritableProvider(const std::string& /*fname*/, IOStatus GetWritableProvider(const std::string& /*fname*/,
EncryptionProvider** result) { EncryptionProvider** result) {
if (provider_) { if (provider_) {
*result = provider_.get(); *result = provider_.get();
return Status::OK(); return IOStatus::OK();
} else { } else {
*result = nullptr; *result = nullptr;
return Status::NotFound("No WriteProvider specified"); return IOStatus::NotFound("No WriteProvider specified");
} }
} }
// Returns the raw encryption provider that should be used to read the input // Returns the raw encryption provider that should be used to read the input
// encrypted file. If there is no such provider, NotFound is returned. // encrypted file. If there is no such provider, NotFound is returned.
Status GetReadableProvider(const std::string& /*fname*/, IOStatus GetReadableProvider(const std::string& /*fname*/,
EncryptionProvider** result) { EncryptionProvider** result) {
if (provider_) { if (provider_) {
*result = provider_.get(); *result = provider_.get();
return Status::OK(); return IOStatus::OK();
} else { } else {
*result = nullptr; *result = nullptr;
return Status::NotFound("No Provider specified"); return IOStatus::NotFound("No Provider specified");
} }
} }
@ -453,13 +510,13 @@ class EncryptedEnvImpl : public EnvWrapper {
// should be encrypted // should be encrypted
// @return OK on success, non-OK on failure. // @return OK on success, non-OK on failure.
template <class TypeFile> template <class TypeFile>
Status CreateWritableCipherStream( IOStatus CreateWritableCipherStream(
const std::string& fname, const std::unique_ptr<TypeFile>& underlying, const std::string& fname, const std::unique_ptr<TypeFile>& underlying,
const EnvOptions& options, size_t* prefix_length, const FileOptions& options, size_t* prefix_length,
std::unique_ptr<BlockAccessCipherStream>* stream) { std::unique_ptr<BlockAccessCipherStream>* stream, IODebugContext* dbg) {
EncryptionProvider* provider = nullptr; EncryptionProvider* provider = nullptr;
*prefix_length = 0; *prefix_length = 0;
Status status = GetWritableProvider(fname, &provider); IOStatus status = GetWritableProvider(fname, &provider);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} else if (provider != nullptr) { } else if (provider != nullptr) {
@ -471,34 +528,36 @@ class EncryptedEnvImpl : public EnvWrapper {
// Initialize prefix // Initialize prefix
buffer.Alignment(underlying->GetRequiredBufferAlignment()); buffer.Alignment(underlying->GetRequiredBufferAlignment());
buffer.AllocateNewBuffer(*prefix_length); buffer.AllocateNewBuffer(*prefix_length);
status = provider->CreateNewPrefix(fname, buffer.BufferStart(), status = status_to_io_status(provider->CreateNewPrefix(
*prefix_length); fname, buffer.BufferStart(), *prefix_length));
if (status.ok()) { if (status.ok()) {
buffer.Size(*prefix_length); buffer.Size(*prefix_length);
prefix = Slice(buffer.BufferStart(), buffer.CurrentSize()); prefix = Slice(buffer.BufferStart(), buffer.CurrentSize());
// Write prefix // Write prefix
status = underlying->Append(prefix); status = underlying->Append(prefix, options.io_options, dbg);
} }
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
} }
// Create cipher stream // Create cipher stream
status = provider->CreateCipherStream(fname, options, prefix, stream); status = status_to_io_status(
provider->CreateCipherStream(fname, options, prefix, stream));
} }
return status; return status;
} }
template <class TypeFile> template <class TypeFile>
Status CreateWritableEncryptedFile(const std::string& fname, IOStatus CreateWritableEncryptedFile(const std::string& fname,
std::unique_ptr<TypeFile>& underlying, std::unique_ptr<TypeFile>& underlying,
const EnvOptions& options, const FileOptions& options,
std::unique_ptr<TypeFile>* result) { std::unique_ptr<TypeFile>* result,
IODebugContext* dbg) {
// Create cipher stream // Create cipher stream
std::unique_ptr<BlockAccessCipherStream> stream; std::unique_ptr<BlockAccessCipherStream> stream;
size_t prefix_length; size_t prefix_length;
Status status = CreateWritableCipherStream(fname, underlying, options, IOStatus status = CreateWritableCipherStream(fname, underlying, options,
&prefix_length, &stream); &prefix_length, &stream, dbg);
if (status.ok()) { if (status.ok()) {
if (stream) { if (stream) {
result->reset(new EncryptedWritableFile( result->reset(new EncryptedWritableFile(
@ -522,15 +581,15 @@ class EncryptedEnvImpl : public EnvWrapper {
// should be encrypted // should be encrypted
// @return OK on success, non-OK on failure. // @return OK on success, non-OK on failure.
template <class TypeFile> template <class TypeFile>
Status CreateRandomWriteCipherStream( IOStatus CreateRandomWriteCipherStream(
const std::string& fname, const std::unique_ptr<TypeFile>& underlying, const std::string& fname, const std::unique_ptr<TypeFile>& underlying,
const EnvOptions& options, size_t* prefix_length, const FileOptions& options, size_t* prefix_length,
std::unique_ptr<BlockAccessCipherStream>* stream) { std::unique_ptr<BlockAccessCipherStream>* stream, IODebugContext* dbg) {
EncryptionProvider* provider = nullptr; EncryptionProvider* provider = nullptr;
*prefix_length = 0; *prefix_length = 0;
Status status = GetWritableProvider(fname, &provider); IOStatus io_s = GetWritableProvider(fname, &provider);
if (!status.ok()) { if (!io_s.ok()) {
return status; return io_s;
} else if (provider != nullptr) { } else if (provider != nullptr) {
// Initialize & write prefix (if needed) // Initialize & write prefix (if needed)
AlignedBuffer buffer; AlignedBuffer buffer;
@ -540,22 +599,23 @@ class EncryptedEnvImpl : public EnvWrapper {
// Initialize prefix // Initialize prefix
buffer.Alignment(underlying->GetRequiredBufferAlignment()); buffer.Alignment(underlying->GetRequiredBufferAlignment());
buffer.AllocateNewBuffer(*prefix_length); buffer.AllocateNewBuffer(*prefix_length);
status = provider->CreateNewPrefix(fname, buffer.BufferStart(), io_s = status_to_io_status(provider->CreateNewPrefix(
*prefix_length); fname, buffer.BufferStart(), *prefix_length));
if (status.ok()) { if (io_s.ok()) {
buffer.Size(*prefix_length); buffer.Size(*prefix_length);
prefix = Slice(buffer.BufferStart(), buffer.CurrentSize()); prefix = Slice(buffer.BufferStart(), buffer.CurrentSize());
// Write prefix // Write prefix
status = underlying->Write(0, prefix); io_s = underlying->Write(0, prefix, options.io_options, dbg);
} }
if (!status.ok()) { if (!io_s.ok()) {
return status; return io_s;
} }
} }
// Create cipher stream // Create cipher stream
status = provider->CreateCipherStream(fname, options, prefix, stream); io_s = status_to_io_status(
provider->CreateCipherStream(fname, options, prefix, stream));
} }
return status; return io_s;
} }
// Creates a CipherStream for the underlying file/name using the options // Creates a CipherStream for the underlying file/name using the options
@ -570,10 +630,10 @@ class EncryptedEnvImpl : public EnvWrapper {
// is encrypted // is encrypted
// @return OK on success, non-OK on failure. // @return OK on success, non-OK on failure.
template <class TypeFile> template <class TypeFile>
Status CreateSequentialCipherStream( IOStatus CreateSequentialCipherStream(
const std::string& fname, const std::unique_ptr<TypeFile>& underlying, const std::string& fname, const std::unique_ptr<TypeFile>& underlying,
const EnvOptions& options, size_t* prefix_length, const FileOptions& options, size_t* prefix_length,
std::unique_ptr<BlockAccessCipherStream>* stream) { std::unique_ptr<BlockAccessCipherStream>* stream, IODebugContext* dbg) {
// Read prefix (if needed) // Read prefix (if needed)
AlignedBuffer buffer; AlignedBuffer buffer;
Slice prefix; Slice prefix;
@ -582,14 +642,15 @@ class EncryptedEnvImpl : public EnvWrapper {
// Read prefix // Read prefix
buffer.Alignment(underlying->GetRequiredBufferAlignment()); buffer.Alignment(underlying->GetRequiredBufferAlignment());
buffer.AllocateNewBuffer(*prefix_length); buffer.AllocateNewBuffer(*prefix_length);
Status status = IOStatus status = underlying->Read(*prefix_length, options.io_options,
underlying->Read(*prefix_length, &prefix, buffer.BufferStart()); &prefix, buffer.BufferStart(), dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
buffer.Size(*prefix_length); buffer.Size(*prefix_length);
} }
return provider_->CreateCipherStream(fname, options, prefix, stream); return status_to_io_status(
provider_->CreateCipherStream(fname, options, prefix, stream));
} }
// Creates a CipherStream for the underlying file/name using the options // Creates a CipherStream for the underlying file/name using the options
@ -604,10 +665,10 @@ class EncryptedEnvImpl : public EnvWrapper {
// is encrypted // is encrypted
// @return OK on success, non-OK on failure. // @return OK on success, non-OK on failure.
template <class TypeFile> template <class TypeFile>
Status CreateRandomReadCipherStream( IOStatus CreateRandomReadCipherStream(
const std::string& fname, const std::unique_ptr<TypeFile>& underlying, const std::string& fname, const std::unique_ptr<TypeFile>& underlying,
const EnvOptions& options, size_t* prefix_length, const FileOptions& options, size_t* prefix_length,
std::unique_ptr<BlockAccessCipherStream>* stream) { std::unique_ptr<BlockAccessCipherStream>* stream, IODebugContext* dbg) {
// Read prefix (if needed) // Read prefix (if needed)
AlignedBuffer buffer; AlignedBuffer buffer;
Slice prefix; Slice prefix;
@ -616,39 +677,48 @@ class EncryptedEnvImpl : public EnvWrapper {
// Read prefix // Read prefix
buffer.Alignment(underlying->GetRequiredBufferAlignment()); buffer.Alignment(underlying->GetRequiredBufferAlignment());
buffer.AllocateNewBuffer(*prefix_length); buffer.AllocateNewBuffer(*prefix_length);
Status status = IOStatus status = underlying->Read(0, *prefix_length, options.io_options,
underlying->Read(0, *prefix_length, &prefix, buffer.BufferStart()); &prefix, buffer.BufferStart(), dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
buffer.Size(*prefix_length); buffer.Size(*prefix_length);
} }
return provider_->CreateCipherStream(fname, options, prefix, stream); return status_to_io_status(
provider_->CreateCipherStream(fname, options, prefix, stream));
} }
public: public:
EncryptedEnvImpl(Env* base_env, EncryptedFileSystemImpl(const std::shared_ptr<FileSystem>& base,
const std::shared_ptr<EncryptionProvider>& provider) const std::shared_ptr<EncryptionProvider>& provider)
: EnvWrapper(base_env) { : EncryptedFileSystem(base) {
provider_ = provider; provider_ = provider;
} }
Status AddCipher(const std::string& descriptor, const char* cipher,
size_t len, bool for_write) override {
return provider_->AddCipher(descriptor, cipher, len, for_write);
}
// NewSequentialFile opens a file for sequential reading. // NewSequentialFile opens a file for sequential reading.
virtual Status NewSequentialFile(const std::string& fname, IOStatus NewSequentialFile(const std::string& fname,
std::unique_ptr<SequentialFile>* result, const FileOptions& options,
const EnvOptions& options) override { std::unique_ptr<FSSequentialFile>* result,
IODebugContext* dbg) override {
result->reset(); result->reset();
if (options.use_mmap_reads) { if (options.use_mmap_reads) {
return Status::InvalidArgument(); return IOStatus::InvalidArgument();
} }
// Open file using underlying Env implementation // Open file using underlying Env implementation
std::unique_ptr<SequentialFile> underlying; std::unique_ptr<FSSequentialFile> underlying;
auto status = EnvWrapper::NewSequentialFile(fname, &underlying, options); auto status =
FileSystemWrapper::NewSequentialFile(fname, options, &underlying, dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
uint64_t file_size; uint64_t file_size;
status = EnvWrapper::GetFileSize(fname, &file_size); status = FileSystemWrapper::GetFileSize(fname, options.io_options,
&file_size, dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
@ -660,7 +730,7 @@ class EncryptedEnvImpl : public EnvWrapper {
std::unique_ptr<BlockAccessCipherStream> stream; std::unique_ptr<BlockAccessCipherStream> stream;
size_t prefix_length; size_t prefix_length;
status = CreateSequentialCipherStream(fname, underlying, options, status = CreateSequentialCipherStream(fname, underlying, options,
&prefix_length, &stream); &prefix_length, &stream, dbg);
if (status.ok()) { if (status.ok()) {
result->reset(new EncryptedSequentialFile( result->reset(new EncryptedSequentialFile(
std::move(underlying), std::move(stream), prefix_length)); std::move(underlying), std::move(stream), prefix_length));
@ -669,23 +739,25 @@ class EncryptedEnvImpl : public EnvWrapper {
} }
// NewRandomAccessFile opens a file for random read access. // NewRandomAccessFile opens a file for random read access.
virtual Status NewRandomAccessFile(const std::string& fname, IOStatus NewRandomAccessFile(const std::string& fname,
std::unique_ptr<RandomAccessFile>* result, const FileOptions& options,
const EnvOptions& options) override { std::unique_ptr<FSRandomAccessFile>* result,
IODebugContext* dbg) override {
result->reset(); result->reset();
if (options.use_mmap_reads) { if (options.use_mmap_reads) {
return Status::InvalidArgument(); return IOStatus::InvalidArgument();
} }
// Open file using underlying Env implementation // Open file using underlying Env implementation
std::unique_ptr<RandomAccessFile> underlying; std::unique_ptr<FSRandomAccessFile> underlying;
auto status = EnvWrapper::NewRandomAccessFile(fname, &underlying, options); auto status = FileSystemWrapper::NewRandomAccessFile(fname, options,
&underlying, dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
std::unique_ptr<BlockAccessCipherStream> stream; std::unique_ptr<BlockAccessCipherStream> stream;
size_t prefix_length; size_t prefix_length;
status = CreateRandomReadCipherStream(fname, underlying, options, status = CreateRandomReadCipherStream(fname, underlying, options,
&prefix_length, &stream); &prefix_length, &stream, dbg);
if (status.ok()) { if (status.ok()) {
if (stream) { if (stream) {
result->reset(new EncryptedRandomAccessFile( result->reset(new EncryptedRandomAccessFile(
@ -698,20 +770,21 @@ class EncryptedEnvImpl : public EnvWrapper {
} }
// NewWritableFile opens a file for sequential writing. // NewWritableFile opens a file for sequential writing.
virtual Status NewWritableFile(const std::string& fname, IOStatus NewWritableFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<WritableFile>* result, std::unique_ptr<FSWritableFile>* result,
const EnvOptions& options) override { IODebugContext* dbg) override {
result->reset(); result->reset();
if (options.use_mmap_writes) { if (options.use_mmap_writes) {
return Status::InvalidArgument(); return IOStatus::InvalidArgument();
} }
// Open file using underlying Env implementation // Open file using underlying Env implementation
std::unique_ptr<WritableFile> underlying; std::unique_ptr<FSWritableFile> underlying;
Status status = EnvWrapper::NewWritableFile(fname, &underlying, options); IOStatus status =
FileSystemWrapper::NewWritableFile(fname, options, &underlying, dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
return CreateWritableEncryptedFile(fname, underlying, options, result); return CreateWritableEncryptedFile(fname, underlying, options, result, dbg);
} }
// Create an object that writes to a new file with the specified // Create an object that writes to a new file with the specified
@ -721,39 +794,42 @@ class EncryptedEnvImpl : public EnvWrapper {
// returns non-OK. // returns non-OK.
// //
// The returned file will only be accessed by one thread at a time. // The returned file will only be accessed by one thread at a time.
virtual Status ReopenWritableFile(const std::string& fname, IOStatus ReopenWritableFile(const std::string& fname,
std::unique_ptr<WritableFile>* result, const FileOptions& options,
const EnvOptions& options) override { std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override {
result->reset(); result->reset();
if (options.use_mmap_writes) { if (options.use_mmap_writes) {
return Status::InvalidArgument(); return IOStatus::InvalidArgument();
} }
// Open file using underlying Env implementation // Open file using underlying Env implementation
std::unique_ptr<WritableFile> underlying; std::unique_ptr<FSWritableFile> underlying;
Status status = EnvWrapper::ReopenWritableFile(fname, &underlying, options); IOStatus status =
FileSystemWrapper::ReopenWritableFile(fname, options, &underlying, dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
return CreateWritableEncryptedFile(fname, underlying, options, result); return CreateWritableEncryptedFile(fname, underlying, options, result, dbg);
} }
// Reuse an existing file by renaming it and opening it as writable. // Reuse an existing file by renaming it and opening it as writable.
virtual Status ReuseWritableFile(const std::string& fname, IOStatus ReuseWritableFile(const std::string& fname,
const std::string& old_fname, const std::string& old_fname,
std::unique_ptr<WritableFile>* result, const FileOptions& options,
const EnvOptions& options) override { std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override {
result->reset(); result->reset();
if (options.use_mmap_writes) { if (options.use_mmap_writes) {
return Status::InvalidArgument(); return IOStatus::InvalidArgument();
} }
// Open file using underlying Env implementation // Open file using underlying Env implementation
std::unique_ptr<WritableFile> underlying; std::unique_ptr<FSWritableFile> underlying;
Status status = auto status = FileSystemWrapper::ReuseWritableFile(
EnvWrapper::ReuseWritableFile(fname, old_fname, &underlying, options); fname, old_fname, options, &underlying, dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
return CreateWritableEncryptedFile(fname, underlying, options, result); return CreateWritableEncryptedFile(fname, underlying, options, result, dbg);
} }
// Open `fname` for random read and write, if file doesn't exist the file // Open `fname` for random read and write, if file doesn't exist the file
@ -761,19 +837,20 @@ class EncryptedEnvImpl : public EnvWrapper {
// *result and returns OK. On failure returns non-OK. // *result and returns OK. On failure returns non-OK.
// //
// The returned file will only be accessed by one thread at a time. // The returned file will only be accessed by one thread at a time.
virtual Status NewRandomRWFile(const std::string& fname, IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<RandomRWFile>* result, std::unique_ptr<FSRandomRWFile>* result,
const EnvOptions& options) override { IODebugContext* dbg) override {
result->reset(); result->reset();
if (options.use_mmap_reads || options.use_mmap_writes) { if (options.use_mmap_reads || options.use_mmap_writes) {
return Status::InvalidArgument(); return IOStatus::InvalidArgument();
} }
// Check file exists // Check file exists
bool isNewFile = !FileExists(fname).ok(); bool isNewFile = !FileExists(fname, options.io_options, dbg).ok();
// Open file using underlying Env implementation // Open file using underlying Env implementation
std::unique_ptr<RandomRWFile> underlying; std::unique_ptr<FSRandomRWFile> underlying;
Status status = EnvWrapper::NewRandomRWFile(fname, &underlying, options); auto status =
FileSystemWrapper::NewRandomRWFile(fname, options, &underlying, dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
@ -783,10 +860,10 @@ class EncryptedEnvImpl : public EnvWrapper {
if (!isNewFile) { if (!isNewFile) {
// File already exists, read prefix // File already exists, read prefix
status = CreateRandomReadCipherStream(fname, underlying, options, status = CreateRandomReadCipherStream(fname, underlying, options,
&prefix_length, &stream); &prefix_length, &stream, dbg);
} else { } else {
status = CreateRandomWriteCipherStream(fname, underlying, options, status = CreateRandomWriteCipherStream(fname, underlying, options,
&prefix_length, &stream); &prefix_length, &stream, dbg);
} }
if (status.ok()) { if (status.ok()) {
if (stream) { if (stream) {
@ -813,9 +890,12 @@ class EncryptedEnvImpl : public EnvWrapper {
// have // have
// permission to access "dir", or if "dir" is invalid. // permission to access "dir", or if "dir" is invalid.
// IOError if an IO Error was encountered // IOError if an IO Error was encountered
virtual Status GetChildrenFileAttributes( IOStatus GetChildrenFileAttributes(const std::string& dir,
const std::string& dir, std::vector<FileAttributes>* result) override { const IOOptions& options,
auto status = EnvWrapper::GetChildrenFileAttributes(dir, result); std::vector<FileAttributes>* result,
IODebugContext* dbg) override {
auto status =
FileSystemWrapper::GetChildrenFileAttributes(dir, options, result, dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
@ -833,13 +913,14 @@ class EncryptedEnvImpl : public EnvWrapper {
it->size_bytes -= provider->GetPrefixLength(); it->size_bytes -= provider->GetPrefixLength();
} }
} }
return Status::OK(); return IOStatus::OK();
} }
// Store the size of fname in *file_size. // Store the size of fname in *file_size.
virtual Status GetFileSize(const std::string& fname, IOStatus GetFileSize(const std::string& fname, const IOOptions& options,
uint64_t* file_size) override { uint64_t* file_size, IODebugContext* dbg) override {
auto status = EnvWrapper::GetFileSize(fname, file_size); auto status =
FileSystemWrapper::GetFileSize(fname, options, file_size, dbg);
if (!status.ok() || !(*file_size)) { if (!status.ok() || !(*file_size)) {
return status; return status;
} }
@ -856,12 +937,19 @@ class EncryptedEnvImpl : public EnvWrapper {
private: private:
std::shared_ptr<EncryptionProvider> provider_; std::shared_ptr<EncryptionProvider> provider_;
}; };
} // namespace
std::shared_ptr<FileSystem> NewEncryptedFS(
const std::shared_ptr<FileSystem>& base,
const std::shared_ptr<EncryptionProvider>& provider) {
return std::make_shared<EncryptedFileSystemImpl>(base, provider);
}
// Returns an Env that encrypts data when stored on disk and decrypts data when // Returns an Env that encrypts data when stored on disk and decrypts data when
// read from disk. // read from disk.
Env* NewEncryptedEnv(Env* base_env, Env* NewEncryptedEnv(Env* base_env,
const std::shared_ptr<EncryptionProvider>& provider) { const std::shared_ptr<EncryptionProvider>& provider) {
return new EncryptedEnvImpl(base_env, provider); return new CompositeEnvWrapper(
base_env, NewEncryptedFS(base_env->GetFileSystem(), provider));
} }
// Encrypt one or more (partial) blocks of data at the file offset. // Encrypt one or more (partial) blocks of data at the file offset.

133
include/rocksdb/env_encryption.h
Unescape View File

@ -10,6 +10,7 @@
#include <string> #include <string>
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/file_system.h"
#include "rocksdb/rocksdb_namespace.h" #include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
@ -171,9 +172,9 @@ class EncryptionProvider {
virtual Status TEST_Initialize() { return Status::OK(); } virtual Status TEST_Initialize() { return Status::OK(); }
}; };
class EncryptedSequentialFile : public SequentialFile { class EncryptedSequentialFile : public FSSequentialFile {
protected: protected:
std::unique_ptr<SequentialFile> file_; std::unique_ptr<FSSequentialFile> file_;
std::unique_ptr<BlockAccessCipherStream> stream_; std::unique_ptr<BlockAccessCipherStream> stream_;
uint64_t offset_; uint64_t offset_;
size_t prefixLength_; size_t prefixLength_;
@ -181,7 +182,7 @@ class EncryptedSequentialFile : public SequentialFile {
public: public:
// Default ctor. Given underlying sequential file is supposed to be at // Default ctor. Given underlying sequential file is supposed to be at
// offset == prefixLength. // offset == prefixLength.
EncryptedSequentialFile(std::unique_ptr<SequentialFile>&& f, EncryptedSequentialFile(std::unique_ptr<FSSequentialFile>&& f,
std::unique_ptr<BlockAccessCipherStream>&& s, std::unique_ptr<BlockAccessCipherStream>&& s,
size_t prefixLength) size_t prefixLength)
: file_(std::move(f)), : file_(std::move(f)),
@ -197,7 +198,8 @@ class EncryptedSequentialFile : public SequentialFile {
// If an error was encountered, returns a non-OK status. // If an error was encountered, returns a non-OK status.
// //
// REQUIRES: External synchronization // REQUIRES: External synchronization
virtual Status Read(size_t n, Slice* result, char* scratch) override; IOStatus Read(size_t n, const IOOptions& options, Slice* result,
char* scratch, IODebugContext* dbg) override;
// Skip "n" bytes from the file. This is guaranteed to be no // Skip "n" bytes from the file. This is guaranteed to be no
// slower that reading the same data, but may be faster. // slower that reading the same data, but may be faster.
@ -206,36 +208,37 @@ class EncryptedSequentialFile : public SequentialFile {
// file, and Skip will return OK. // file, and Skip will return OK.
// //
// REQUIRES: External synchronization // REQUIRES: External synchronization
virtual Status Skip(uint64_t n) override; IOStatus Skip(uint64_t n) override;
// Indicates the upper layers if the current SequentialFile implementation // Indicates the upper layers if the current SequentialFile implementation
// uses direct IO. // uses direct IO.
virtual bool use_direct_io() const override; bool use_direct_io() const override;
// Use the returned alignment value to allocate // Use the returned alignment value to allocate
// aligned buffer for Direct I/O // aligned buffer for Direct I/O
virtual size_t GetRequiredBufferAlignment() const override; size_t GetRequiredBufferAlignment() const override;
// Remove any kind of caching of data from the offset to offset+length // Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file. // of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop. // If the system is not caching the file contents, then this is a noop.
virtual Status InvalidateCache(size_t offset, size_t length) override; IOStatus InvalidateCache(size_t offset, size_t length) override;
// Positioned Read for direct I/O // Positioned Read for direct I/O
// If Direct I/O enabled, offset, n, and scratch should be properly aligned // If Direct I/O enabled, offset, n, and scratch should be properly aligned
virtual Status PositionedRead(uint64_t offset, size_t n, Slice* result, IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& options,
char* scratch) override; Slice* result, char* scratch,
IODebugContext* dbg) override;
}; };
// A file abstraction for randomly reading the contents of a file. // A file abstraction for randomly reading the contents of a file.
class EncryptedRandomAccessFile : public RandomAccessFile { class EncryptedRandomAccessFile : public FSRandomAccessFile {
protected: protected:
std::unique_ptr<RandomAccessFile> file_; std::unique_ptr<FSRandomAccessFile> file_;
std::unique_ptr<BlockAccessCipherStream> stream_; std::unique_ptr<BlockAccessCipherStream> stream_;
size_t prefixLength_; size_t prefixLength_;
public: public:
EncryptedRandomAccessFile(std::unique_ptr<RandomAccessFile>&& f, EncryptedRandomAccessFile(std::unique_ptr<FSRandomAccessFile>&& f,
std::unique_ptr<BlockAccessCipherStream>&& s, std::unique_ptr<BlockAccessCipherStream>&& s,
size_t prefixLength) size_t prefixLength)
: file_(std::move(f)), : file_(std::move(f)),
@ -252,11 +255,13 @@ class EncryptedRandomAccessFile : public RandomAccessFile {
// //
// Safe for concurrent use by multiple threads. // Safe for concurrent use by multiple threads.
// If Direct I/O enabled, offset, n, and scratch should be aligned properly. // If Direct I/O enabled, offset, n, and scratch should be aligned properly.
virtual Status Read(uint64_t offset, size_t n, Slice* result, IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
char* scratch) const override; Slice* result, char* scratch,
IODebugContext* dbg) const override;
// Readahead the file starting from offset by n bytes for caching. // Readahead the file starting from offset by n bytes for caching.
virtual Status Prefetch(uint64_t offset, size_t n) override; IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options,
IODebugContext* dbg) override;
// Tries to get an unique ID for this file that will be the same each time // Tries to get an unique ID for this file that will be the same each time
// the file is opened (and will stay the same while the file is open). // the file is opened (and will stay the same while the file is open).
@ -273,71 +278,76 @@ class EncryptedRandomAccessFile : public RandomAccessFile {
// a single varint. // a single varint.
// //
// Note: these IDs are only valid for the duration of the process. // Note: these IDs are only valid for the duration of the process.
virtual size_t GetUniqueId(char* id, size_t max_size) const override; size_t GetUniqueId(char* id, size_t max_size) const override;
virtual void Hint(AccessPattern pattern) override; void Hint(AccessPattern pattern) override;
// Indicates the upper layers if the current RandomAccessFile implementation // Indicates the upper layers if the current RandomAccessFile implementation
// uses direct IO. // uses direct IO.
virtual bool use_direct_io() const override; bool use_direct_io() const override;
// Use the returned alignment value to allocate // Use the returned alignment value to allocate
// aligned buffer for Direct I/O // aligned buffer for Direct I/O
virtual size_t GetRequiredBufferAlignment() const override; size_t GetRequiredBufferAlignment() const override;
// Remove any kind of caching of data from the offset to offset+length // Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file. // of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop. // If the system is not caching the file contents, then this is a noop.
virtual Status InvalidateCache(size_t offset, size_t length) override; IOStatus InvalidateCache(size_t offset, size_t length) override;
}; };
// A file abstraction for sequential writing. The implementation // A file abstraction for sequential writing. The implementation
// must provide buffering since callers may append small fragments // must provide buffering since callers may append small fragments
// at a time to the file. // at a time to the file.
class EncryptedWritableFile : public WritableFileWrapper { class EncryptedWritableFile : public FSWritableFile {
protected: protected:
std::unique_ptr<WritableFile> file_; std::unique_ptr<FSWritableFile> file_;
std::unique_ptr<BlockAccessCipherStream> stream_; std::unique_ptr<BlockAccessCipherStream> stream_;
size_t prefixLength_; size_t prefixLength_;
public: public:
// Default ctor. Prefix is assumed to be written already. // Default ctor. Prefix is assumed to be written already.
EncryptedWritableFile(std::unique_ptr<WritableFile>&& f, EncryptedWritableFile(std::unique_ptr<FSWritableFile>&& f,
std::unique_ptr<BlockAccessCipherStream>&& s, std::unique_ptr<BlockAccessCipherStream>&& s,
size_t prefixLength) size_t prefixLength)
: WritableFileWrapper(f.get()), : file_(std::move(f)),
file_(std::move(f)),
stream_(std::move(s)), stream_(std::move(s)),
prefixLength_(prefixLength) {} prefixLength_(prefixLength) {}
Status Append(const Slice& data) override; using FSWritableFile::Append;
IOStatus Append(const Slice& data, const IOOptions& options,
IODebugContext* dbg) override;
Status PositionedAppend(const Slice& data, uint64_t offset) override; using FSWritableFile::PositionedAppend;
IOStatus PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& options,
IODebugContext* dbg) override;
// Indicates the upper layers if the current WritableFile implementation // Indicates the upper layers if the current WritableFile implementation
// uses direct IO. // uses direct IO.
virtual bool use_direct_io() const override; bool use_direct_io() const override;
// Use the returned alignment value to allocate // Use the returned alignment value to allocate
// aligned buffer for Direct I/O // aligned buffer for Direct I/O
virtual size_t GetRequiredBufferAlignment() const override; size_t GetRequiredBufferAlignment() const override;
/* /*
* Get the size of valid data in the file. * Get the size of valid data in the file.
*/ */
virtual uint64_t GetFileSize() override; uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) override;
// Truncate is necessary to trim the file to the correct size // Truncate is necessary to trim the file to the correct size
// before closing. It is not always possible to keep track of the file // before closing. It is not always possible to keep track of the file
// size due to whole pages writes. The behavior is undefined if called // size due to whole pages writes. The behavior is undefined if called
// with other writes to follow. // with other writes to follow.
virtual Status Truncate(uint64_t size) override; IOStatus Truncate(uint64_t size, const IOOptions& options,
IODebugContext* dbg) override;
// Remove any kind of caching of data from the offset to offset+length // Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file. // of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop. // If the system is not caching the file contents, then this is a noop.
// This call has no effect on dirty pages in the cache. // This call has no effect on dirty pages in the cache.
virtual Status InvalidateCache(size_t offset, size_t length) override; IOStatus InvalidateCache(size_t offset, size_t length) override;
// Sync a file range with disk. // Sync a file range with disk.
// offset is the starting byte of the file range to be synchronized. // offset is the starting byte of the file range to be synchronized.
@ -345,28 +355,37 @@ class EncryptedWritableFile : public WritableFileWrapper {
// This asks the OS to initiate flushing the cached data to disk, // This asks the OS to initiate flushing the cached data to disk,
// without waiting for completion. // without waiting for completion.
// Default implementation does nothing. // Default implementation does nothing.
virtual Status RangeSync(uint64_t offset, uint64_t nbytes) override; IOStatus RangeSync(uint64_t offset, uint64_t nbytes, const IOOptions& options,
IODebugContext* dbg) override;
// PrepareWrite performs any necessary preparation for a write // PrepareWrite performs any necessary preparation for a write
// before the write actually occurs. This allows for pre-allocation // before the write actually occurs. This allows for pre-allocation
// of space on devices where it can result in less file // of space on devices where it can result in less file
// fragmentation and/or less waste from over-zealous filesystem // fragmentation and/or less waste from over-zealous filesystem
// pre-allocation. // pre-allocation.
virtual void PrepareWrite(size_t offset, size_t len) override; void PrepareWrite(size_t offset, size_t len, const IOOptions& options,
IODebugContext* dbg) override;
// Pre-allocates space for a file. // Pre-allocates space for a file.
virtual Status Allocate(uint64_t offset, uint64_t len) override; IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override;
IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override;
IOStatus Close(const IOOptions& options, IODebugContext* dbg) override;
}; };
// A file abstraction for random reading and writing. // A file abstraction for random reading and writing.
class EncryptedRandomRWFile : public RandomRWFile { class EncryptedRandomRWFile : public FSRandomRWFile {
protected: protected:
std::unique_ptr<RandomRWFile> file_; std::unique_ptr<FSRandomRWFile> file_;
std::unique_ptr<BlockAccessCipherStream> stream_; std::unique_ptr<BlockAccessCipherStream> stream_;
size_t prefixLength_; size_t prefixLength_;
public: public:
EncryptedRandomRWFile(std::unique_ptr<RandomRWFile>&& f, EncryptedRandomRWFile(std::unique_ptr<FSRandomRWFile>&& f,
std::unique_ptr<BlockAccessCipherStream>&& s, std::unique_ptr<BlockAccessCipherStream>&& s,
size_t prefixLength) size_t prefixLength)
: file_(std::move(f)), : file_(std::move(f)),
@ -375,31 +394,49 @@ class EncryptedRandomRWFile : public RandomRWFile {
// Indicates if the class makes use of direct I/O // Indicates if the class makes use of direct I/O
// If false you must pass aligned buffer to Write() // If false you must pass aligned buffer to Write()
virtual bool use_direct_io() const override; bool use_direct_io() const override;
// Use the returned alignment value to allocate // Use the returned alignment value to allocate
// aligned buffer for Direct I/O // aligned buffer for Direct I/O
virtual size_t GetRequiredBufferAlignment() const override; size_t GetRequiredBufferAlignment() const override;
// Write bytes in `data` at offset `offset`, Returns Status::OK() on success. // Write bytes in `data` at offset `offset`, Returns Status::OK() on success.
// Pass aligned buffer when use_direct_io() returns true. // Pass aligned buffer when use_direct_io() returns true.
virtual Status Write(uint64_t offset, const Slice& data) override; IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options,
IODebugContext* dbg) override;
// Read up to `n` bytes starting from offset `offset` and store them in // Read up to `n` bytes starting from offset `offset` and store them in
// result, provided `scratch` size should be at least `n`. // result, provided `scratch` size should be at least `n`.
// Returns Status::OK() on success. // Returns Status::OK() on success.
virtual Status Read(uint64_t offset, size_t n, Slice* result, IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
char* scratch) const override; Slice* result, char* scratch,
IODebugContext* dbg) const override;
virtual Status Flush() override; IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override;
virtual Status Sync() override; IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override;
virtual Status Fsync() override; IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override;
virtual Status Close() override; IOStatus Close(const IOOptions& options, IODebugContext* dbg) override;
}; };
class EncryptedFileSystem : public FileSystemWrapper {
public:
explicit EncryptedFileSystem(const std::shared_ptr<FileSystem>& base)
: FileSystemWrapper(base) {}
// Method to add a new cipher key for use by the EncryptionProvider.
// @param description Descriptor for this key.
// @param cipher The cryptographic key to use
// @param len The length of the cipher key
// @param for_write If true, this cipher should be used for writing files.
// If false, this cipher should only be used for reading
// files
// @return OK if the cipher was successfully added to the provider, non-OK
// otherwise
virtual Status AddCipher(const std::string& descriptor, const char* cipher,
size_t len, bool for_write) = 0;
};
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif // !defined(ROCKSDB_LITE) #endif // !defined(ROCKSDB_LITE)

162
utilities/env_timed.cc
Unescape View File

@ -3,138 +3,176 @@
// COPYING file in the root directory) and Apache 2.0 License // COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#include "env/composite_env_wrapper.h"
#include "monitoring/perf_context_imp.h" #include "monitoring/perf_context_imp.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/file_system.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
namespace {
// An environment that measures function call times for filesystem class TimedFileSystem : public FileSystemWrapper {
// operations, reporting results to variables in PerfContext.
class TimedEnv : public EnvWrapper {
public: public:
explicit TimedEnv(Env* base_env) : EnvWrapper(base_env) {} explicit TimedFileSystem(const std::shared_ptr<FileSystem>& base)
: FileSystemWrapper(base) {}
Status NewSequentialFile(const std::string& fname,
std::unique_ptr<SequentialFile>* result, const char* Name() const override { return "TimedFS"; }
const EnvOptions& options) override { IOStatus NewSequentialFile(const std::string& fname,
const FileOptions& options,
std::unique_ptr<FSSequentialFile>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_sequential_file_nanos); PERF_TIMER_GUARD(env_new_sequential_file_nanos);
return EnvWrapper::NewSequentialFile(fname, result, options); return FileSystemWrapper::NewSequentialFile(fname, options, result, dbg);
} }
Status NewRandomAccessFile(const std::string& fname, IOStatus NewRandomAccessFile(const std::string& fname,
std::unique_ptr<RandomAccessFile>* result, const FileOptions& options,
const EnvOptions& options) override { std::unique_ptr<FSRandomAccessFile>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_random_access_file_nanos); PERF_TIMER_GUARD(env_new_random_access_file_nanos);
return EnvWrapper::NewRandomAccessFile(fname, result, options); return FileSystemWrapper::NewRandomAccessFile(fname, options, result, dbg);
} }
Status NewWritableFile(const std::string& fname, IOStatus NewWritableFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<WritableFile>* result, std::unique_ptr<FSWritableFile>* result,
const EnvOptions& options) override { IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_writable_file_nanos); PERF_TIMER_GUARD(env_new_writable_file_nanos);
return EnvWrapper::NewWritableFile(fname, result, options); return FileSystemWrapper::NewWritableFile(fname, options, result, dbg);
} }
Status ReuseWritableFile(const std::string& fname, IOStatus ReuseWritableFile(const std::string& fname,
const std::string& old_fname, const std::string& old_fname,
std::unique_ptr<WritableFile>* result, const FileOptions& options,
const EnvOptions& options) override { std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_reuse_writable_file_nanos); PERF_TIMER_GUARD(env_reuse_writable_file_nanos);
return EnvWrapper::ReuseWritableFile(fname, old_fname, result, options); return FileSystemWrapper::ReuseWritableFile(fname, old_fname, options,
result, dbg);
} }
Status NewRandomRWFile(const std::string& fname, IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<RandomRWFile>* result, std::unique_ptr<FSRandomRWFile>* result,
const EnvOptions& options) override { IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_random_rw_file_nanos); PERF_TIMER_GUARD(env_new_random_rw_file_nanos);
return EnvWrapper::NewRandomRWFile(fname, result, options); return FileSystemWrapper::NewRandomRWFile(fname, options, result, dbg);
} }
Status NewDirectory(const std::string& name, IOStatus NewDirectory(const std::string& name, const IOOptions& options,
std::unique_ptr<Directory>* result) override { std::unique_ptr<FSDirectory>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_directory_nanos); PERF_TIMER_GUARD(env_new_directory_nanos);
return EnvWrapper::NewDirectory(name, result); return FileSystemWrapper::NewDirectory(name, options, result, dbg);
} }
Status FileExists(const std::string& fname) override { IOStatus FileExists(const std::string& fname, const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_file_exists_nanos); PERF_TIMER_GUARD(env_file_exists_nanos);
return EnvWrapper::FileExists(fname); return FileSystemWrapper::FileExists(fname, options, dbg);
} }
Status GetChildren(const std::string& dir, IOStatus GetChildren(const std::string& dir, const IOOptions& options,
std::vector<std::string>* result) override { std::vector<std::string>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_get_children_nanos); PERF_TIMER_GUARD(env_get_children_nanos);
return EnvWrapper::GetChildren(dir, result); return FileSystemWrapper::GetChildren(dir, options, result, dbg);
} }
Status GetChildrenFileAttributes( IOStatus GetChildrenFileAttributes(const std::string& dir,
const std::string& dir, std::vector<FileAttributes>* result) override { const IOOptions& options,
std::vector<FileAttributes>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_get_children_file_attributes_nanos); PERF_TIMER_GUARD(env_get_children_file_attributes_nanos);
return EnvWrapper::GetChildrenFileAttributes(dir, result); return FileSystemWrapper::GetChildrenFileAttributes(dir, options, result,
dbg);
} }
Status DeleteFile(const std::string& fname) override { IOStatus DeleteFile(const std::string& fname, const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_delete_file_nanos); PERF_TIMER_GUARD(env_delete_file_nanos);
return EnvWrapper::DeleteFile(fname); return FileSystemWrapper::DeleteFile(fname, options, dbg);
} }
Status CreateDir(const std::string& dirname) override { IOStatus CreateDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_create_dir_nanos); PERF_TIMER_GUARD(env_create_dir_nanos);
return EnvWrapper::CreateDir(dirname); return FileSystemWrapper::CreateDir(dirname, options, dbg);
} }
Status CreateDirIfMissing(const std::string& dirname) override { IOStatus CreateDirIfMissing(const std::string& dirname,
const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_create_dir_if_missing_nanos); PERF_TIMER_GUARD(env_create_dir_if_missing_nanos);
return EnvWrapper::CreateDirIfMissing(dirname); return FileSystemWrapper::CreateDirIfMissing(dirname, options, dbg);
} }
Status DeleteDir(const std::string& dirname) override { IOStatus DeleteDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_delete_dir_nanos); PERF_TIMER_GUARD(env_delete_dir_nanos);
return EnvWrapper::DeleteDir(dirname); return FileSystemWrapper::DeleteDir(dirname, options, dbg);
} }
Status GetFileSize(const std::string& fname, uint64_t* file_size) override { IOStatus GetFileSize(const std::string& fname, const IOOptions& options,
uint64_t* file_size, IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_get_file_size_nanos); PERF_TIMER_GUARD(env_get_file_size_nanos);
return EnvWrapper::GetFileSize(fname, file_size); return FileSystemWrapper::GetFileSize(fname, options, file_size, dbg);
} }
Status GetFileModificationTime(const std::string& fname, IOStatus GetFileModificationTime(const std::string& fname,
uint64_t* file_mtime) override { const IOOptions& options,
uint64_t* file_mtime,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_get_file_modification_time_nanos); PERF_TIMER_GUARD(env_get_file_modification_time_nanos);
return EnvWrapper::GetFileModificationTime(fname, file_mtime); return FileSystemWrapper::GetFileModificationTime(fname, options,
file_mtime, dbg);
} }
Status RenameFile(const std::string& src, const std::string& dst) override { IOStatus RenameFile(const std::string& src, const std::string& dst,
const IOOptions& options, IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_rename_file_nanos); PERF_TIMER_GUARD(env_rename_file_nanos);
return EnvWrapper::RenameFile(src, dst); return FileSystemWrapper::RenameFile(src, dst, options, dbg);
} }
Status LinkFile(const std::string& src, const std::string& dst) override { IOStatus LinkFile(const std::string& src, const std::string& dst,
const IOOptions& options, IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_link_file_nanos); PERF_TIMER_GUARD(env_link_file_nanos);
return EnvWrapper::LinkFile(src, dst); return FileSystemWrapper::LinkFile(src, dst, options, dbg);
} }
Status LockFile(const std::string& fname, FileLock** lock) override { IOStatus LockFile(const std::string& fname, const IOOptions& options,
FileLock** lock, IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_lock_file_nanos); PERF_TIMER_GUARD(env_lock_file_nanos);
return EnvWrapper::LockFile(fname, lock); return FileSystemWrapper::LockFile(fname, options, lock, dbg);
} }
Status UnlockFile(FileLock* lock) override { IOStatus UnlockFile(FileLock* lock, const IOOptions& options,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_unlock_file_nanos); PERF_TIMER_GUARD(env_unlock_file_nanos);
return EnvWrapper::UnlockFile(lock); return FileSystemWrapper::UnlockFile(lock, options, dbg);
} }
Status NewLogger(const std::string& fname, IOStatus NewLogger(const std::string& fname, const IOOptions& options,
std::shared_ptr<Logger>* result) override { std::shared_ptr<Logger>* result,
IODebugContext* dbg) override {
PERF_TIMER_GUARD(env_new_logger_nanos); PERF_TIMER_GUARD(env_new_logger_nanos);
return EnvWrapper::NewLogger(fname, result); return FileSystemWrapper::NewLogger(fname, options, result, dbg);
} }
}; };
} // namespace
Env* NewTimedEnv(Env* base_env) { return new TimedEnv(base_env); } std::shared_ptr<FileSystem> NewTimedFileSystem(
const std::shared_ptr<FileSystem>& base) {
return std::make_shared<TimedFileSystem>(base);
}
// An environment that measures function call times for filesystem
// operations, reporting results to variables in PerfContext.
Env* NewTimedEnv(Env* base_env) {
std::shared_ptr<FileSystem> timed_fs =
NewTimedFileSystem(base_env->GetFileSystem());
return new CompositeEnvWrapper(base_env, timed_fs);
}
#else // ROCKSDB_LITE #else // ROCKSDB_LITE

Write Preview
Loading…
Cancel
Save