NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove duplicate comments in EncryptedEnv (#11549)

Browse Source 
Summary:
There are some comments on subclasses in EncryptedEnv module which are duplicate to their parent classes, it would be nice to remove the duplication and keep the consistency if the comments on parent classes updated in someday.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/11549

Reviewed By: akankshamahajan15

Differential Revision: D47007061

Pulled By: ajkr

fbshipit-source-id: 8bfdaf9f2418a24ca951c30bb88e90ac861d9016
oxigraph-main
Yingchun Lai 2 years ago committed by Facebook GitHub Bot
parent b14c0b0602
commit 44524cf5da
3 changed files with 20 additions and 291 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 176
      env/env_encryption.cc
  2. 17
      env/env_encryption_ctr.h
  3. 118
      include/rocksdb/env_encryption.h

176
env/env_encryption.cc vendored
Unescape View File

@ -30,14 +30,6 @@ std::shared_ptr<EncryptionProvider> EncryptionProvider::NewCTRProvider(
return std::make_shared<CTREncryptionProvider>(cipher); return std::make_shared<CTREncryptionProvider>(cipher);
} }
// Read up to "n" bytes from the file. "scratch[0..n-1]" may be
// written by this routine. Sets "*result" to the data that was
// read (including if fewer than "n" bytes were successfully read).
// May set "*result" to point at data in "scratch[0..n-1]", so
// "scratch[0..n-1]" must be live when "*result" is used.
// If an error was encountered, returns a non-OK status.
//
// REQUIRES: External synchronization
IOStatus EncryptedSequentialFile::Read(size_t n, const IOOptions& options, IOStatus EncryptedSequentialFile::Read(size_t n, const IOOptions& options,
Slice* result, char* scratch, Slice* result, char* scratch,
IODebugContext* dbg) { IODebugContext* dbg) {
@ -52,19 +44,12 @@ IOStatus EncryptedSequentialFile::Read(size_t n, const IOOptions& options,
stream_->Decrypt(offset_, (char*)result->data(), result->size())); stream_->Decrypt(offset_, (char*)result->data(), result->size()));
} }
if (io_s.ok()) { if (io_s.ok()) {
offset_ += result->size(); // We've already ready data from disk, so update offset_ += result->size(); // We've already read data from disk, so update
// offset_ even if decryption fails. // offset_ even if decryption fails.
} }
return io_s; return io_s;
} }
// Skip "n" bytes from the file. This is guaranteed to be no
// slower that reading the same data, but may be faster.
//
// If end of file is reached, skipping will stop at the end of the
// file, and Skip will return OK.
//
// REQUIRES: External synchronization
IOStatus EncryptedSequentialFile::Skip(uint64_t n) { IOStatus EncryptedSequentialFile::Skip(uint64_t n) {
auto status = file_->Skip(n); auto status = file_->Skip(n);
if (!status.ok()) { if (!status.ok()) {
@ -74,28 +59,19 @@ IOStatus EncryptedSequentialFile::Skip(uint64_t n) {
return status; return status;
} }
// Indicates the upper layers if the current SequentialFile implementation
// uses direct IO.
bool EncryptedSequentialFile::use_direct_io() const { bool EncryptedSequentialFile::use_direct_io() const {
return file_->use_direct_io(); return file_->use_direct_io();
} }
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t EncryptedSequentialFile::GetRequiredBufferAlignment() const { size_t EncryptedSequentialFile::GetRequiredBufferAlignment() const {
return file_->GetRequiredBufferAlignment(); return file_->GetRequiredBufferAlignment();
} }
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
IOStatus EncryptedSequentialFile::InvalidateCache(size_t offset, IOStatus EncryptedSequentialFile::InvalidateCache(size_t offset,
size_t length) { size_t length) {
return file_->InvalidateCache(offset + prefixLength_, length); return file_->InvalidateCache(offset + prefixLength_, length);
} }
// Positioned Read for direct I/O
// If Direct I/O enabled, offset, n, and scratch should be properly aligned
IOStatus EncryptedSequentialFile::PositionedRead(uint64_t offset, size_t n, IOStatus EncryptedSequentialFile::PositionedRead(uint64_t offset, size_t n,
const IOOptions& options, const IOOptions& options,
Slice* result, char* scratch, Slice* result, char* scratch,
@ -115,16 +91,6 @@ IOStatus EncryptedSequentialFile::PositionedRead(uint64_t offset, size_t n,
return io_s; return io_s;
} }
// Read up to "n" bytes from the file starting at "offset".
// "scratch[0..n-1]" may be written by this routine. Sets "*result"
// to the data that was read (including if fewer than "n" bytes were
// successfully read). May set "*result" to point at data in
// "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
// "*result" is used. If an error was encountered, returns a non-OK
// status.
//
// Safe for concurrent use by multiple threads.
// If Direct I/O enabled, offset, n, and scratch should be aligned properly.
IOStatus EncryptedRandomAccessFile::Read(uint64_t offset, size_t n, IOStatus EncryptedRandomAccessFile::Read(uint64_t offset, size_t n,
const IOOptions& options, const IOOptions& options,
Slice* result, char* scratch, Slice* result, char* scratch,
@ -143,29 +109,12 @@ IOStatus EncryptedRandomAccessFile::Read(uint64_t offset, size_t n,
return io_s; return io_s;
} }
// Readahead the file starting from offset by n bytes for caching.
IOStatus EncryptedRandomAccessFile::Prefetch(uint64_t offset, size_t n, IOStatus EncryptedRandomAccessFile::Prefetch(uint64_t offset, size_t n,
const IOOptions& options, const IOOptions& options,
IODebugContext* dbg) { IODebugContext* dbg) {
// return Status::OK();
return file_->Prefetch(offset + prefixLength_, n, options, dbg); return file_->Prefetch(offset + prefixLength_, n, options, dbg);
} }
// Tries to get an unique ID for this file that will be the same each time
// the file is opened (and will stay the same while the file is open).
// Furthermore, it tries to make this ID at most "max_size" bytes. If such an
// ID can be created this function returns the length of the ID and places it
// in "id"; otherwise, this function returns 0, in which case "id"
// may not have been modified.
//
// This function guarantees, for IDs from a given environment, two unique ids
// cannot be made equal to each other by adding arbitrary bytes to one of
// them. That is, no unique ID is the prefix of another.
//
// This function guarantees that the returned ID will not be interpretable as
// a single varint.
//
// Note: these IDs are only valid for the duration of the process.
size_t EncryptedRandomAccessFile::GetUniqueId(char* id, size_t max_size) const { size_t EncryptedRandomAccessFile::GetUniqueId(char* id, size_t max_size) const {
return file_->GetUniqueId(id, max_size); return file_->GetUniqueId(id, max_size);
}; };
@ -174,29 +123,19 @@ void EncryptedRandomAccessFile::Hint(AccessPattern pattern) {
file_->Hint(pattern); file_->Hint(pattern);
} }
// Indicates the upper layers if the current RandomAccessFile implementation
// uses direct IO.
bool EncryptedRandomAccessFile::use_direct_io() const { bool EncryptedRandomAccessFile::use_direct_io() const {
return file_->use_direct_io(); return file_->use_direct_io();
} }
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t EncryptedRandomAccessFile::GetRequiredBufferAlignment() const { size_t EncryptedRandomAccessFile::GetRequiredBufferAlignment() const {
return file_->GetRequiredBufferAlignment(); return file_->GetRequiredBufferAlignment();
} }
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
IOStatus EncryptedRandomAccessFile::InvalidateCache(size_t offset, IOStatus EncryptedRandomAccessFile::InvalidateCache(size_t offset,
size_t length) { size_t length) {
return file_->InvalidateCache(offset + prefixLength_, length); return file_->InvalidateCache(offset + prefixLength_, length);
} }
// A file abstraction for sequential writing. The implementation
// must provide buffering since callers may append small fragments
// at a time to the file.
IOStatus EncryptedWritableFile::Append(const Slice& data, IOStatus EncryptedWritableFile::Append(const Slice& data,
const IOOptions& options, const IOOptions& options,
IODebugContext* dbg) { IODebugContext* dbg) {
@ -252,67 +191,39 @@ IOStatus EncryptedWritableFile::PositionedAppend(const Slice& data,
return file_->PositionedAppend(dataToAppend, offset, options, dbg); return file_->PositionedAppend(dataToAppend, offset, options, dbg);
} }
// Indicates the upper layers if the current WritableFile implementation
// uses direct IO.
bool EncryptedWritableFile::use_direct_io() const { bool EncryptedWritableFile::use_direct_io() const {
return file_->use_direct_io(); return file_->use_direct_io();
} }
// true if Sync() and Fsync() are safe to call concurrently with Append()
// and Flush().
bool EncryptedWritableFile::IsSyncThreadSafe() const { bool EncryptedWritableFile::IsSyncThreadSafe() const {
return file_->IsSyncThreadSafe(); return file_->IsSyncThreadSafe();
} }
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t EncryptedWritableFile::GetRequiredBufferAlignment() const { size_t EncryptedWritableFile::GetRequiredBufferAlignment() const {
return file_->GetRequiredBufferAlignment(); return file_->GetRequiredBufferAlignment();
} }
/*
* Get the size of valid data in the file.
*/
uint64_t EncryptedWritableFile::GetFileSize(const IOOptions& options, uint64_t EncryptedWritableFile::GetFileSize(const IOOptions& options,
IODebugContext* dbg) { IODebugContext* dbg) {
return file_->GetFileSize(options, dbg) - prefixLength_; return file_->GetFileSize(options, dbg) - prefixLength_;
} }
// Truncate is necessary to trim the file to the correct size
// before closing. It is not always possible to keep track of the file
// size due to whole pages writes. The behavior is undefined if called
// with other writes to follow.
IOStatus EncryptedWritableFile::Truncate(uint64_t size, IOStatus EncryptedWritableFile::Truncate(uint64_t size,
const IOOptions& options, const IOOptions& options,
IODebugContext* dbg) { IODebugContext* dbg) {
return file_->Truncate(size + prefixLength_, options, dbg); return file_->Truncate(size + prefixLength_, options, dbg);
} }
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
// This call has no effect on dirty pages in the cache.
IOStatus EncryptedWritableFile::InvalidateCache(size_t offset, size_t length) { IOStatus EncryptedWritableFile::InvalidateCache(size_t offset, size_t length) {
return file_->InvalidateCache(offset + prefixLength_, length); return file_->InvalidateCache(offset + prefixLength_, length);
} }
// Sync a file range with disk.
// offset is the starting byte of the file range to be synchronized.
// nbytes specifies the length of the range to be synchronized.
// This asks the OS to initiate flushing the cached data to disk,
// without waiting for completion.
// Default implementation does nothing.
IOStatus EncryptedWritableFile::RangeSync(uint64_t offset, uint64_t nbytes, IOStatus EncryptedWritableFile::RangeSync(uint64_t offset, uint64_t nbytes,
const IOOptions& options, const IOOptions& options,
IODebugContext* dbg) { IODebugContext* dbg) {
return file_->RangeSync(offset + prefixLength_, nbytes, options, dbg); return file_->RangeSync(offset + prefixLength_, nbytes, options, dbg);
} }
// PrepareWrite performs any necessary preparation for a write
// before the write actually occurs. This allows for pre-allocation
// of space on devices where it can result in less file
// fragmentation and/or less waste from over-zealous filesystem
// pre-allocation.
void EncryptedWritableFile::PrepareWrite(size_t offset, size_t len, void EncryptedWritableFile::PrepareWrite(size_t offset, size_t len,
const IOOptions& options, const IOOptions& options,
IODebugContext* dbg) { IODebugContext* dbg) {
@ -330,7 +241,6 @@ void EncryptedWritableFile::GetPreallocationStatus(
file_->GetPreallocationStatus(block_size, last_allocated_block); file_->GetPreallocationStatus(block_size, last_allocated_block);
} }
// Pre-allocates space for a file.
IOStatus EncryptedWritableFile::Allocate(uint64_t offset, uint64_t len, IOStatus EncryptedWritableFile::Allocate(uint64_t offset, uint64_t len,
const IOOptions& options, const IOOptions& options,
IODebugContext* dbg) { IODebugContext* dbg) {
@ -352,22 +262,14 @@ IOStatus EncryptedWritableFile::Close(const IOOptions& options,
return file_->Close(options, dbg); return file_->Close(options, dbg);
} }
// A file abstraction for random reading and writing.
// Indicates if the class makes use of direct I/O
// If false you must pass aligned buffer to Write()
bool EncryptedRandomRWFile::use_direct_io() const { bool EncryptedRandomRWFile::use_direct_io() const {
return file_->use_direct_io(); return file_->use_direct_io();
} }
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t EncryptedRandomRWFile::GetRequiredBufferAlignment() const { size_t EncryptedRandomRWFile::GetRequiredBufferAlignment() const {
return file_->GetRequiredBufferAlignment(); return file_->GetRequiredBufferAlignment();
} }
// Write bytes in `data` at offset `offset`, Returns Status::OK() on success.
// Pass aligned buffer when use_direct_io() returns true.
IOStatus EncryptedRandomRWFile::Write(uint64_t offset, const Slice& data, IOStatus EncryptedRandomRWFile::Write(uint64_t offset, const Slice& data,
const IOOptions& options, const IOOptions& options,
IODebugContext* dbg) { IODebugContext* dbg) {
@ -394,9 +296,6 @@ IOStatus EncryptedRandomRWFile::Write(uint64_t offset, const Slice& data,
return file_->Write(offset, dataToWrite, options, dbg); return file_->Write(offset, dataToWrite, options, dbg);
} }
// Read up to `n` bytes starting from offset `offset` and store them in
// result, provided `scratch` size should be at least `n`.
// Returns Status::OK() on success.
IOStatus EncryptedRandomRWFile::Read(uint64_t offset, size_t n, IOStatus EncryptedRandomRWFile::Read(uint64_t offset, size_t n,
const IOOptions& options, Slice* result, const IOOptions& options, Slice* result,
char* scratch, IODebugContext* dbg) const { char* scratch, IODebugContext* dbg) const {
@ -678,7 +577,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return provider_->AddCipher(descriptor, cipher, len, for_write); return provider_->AddCipher(descriptor, cipher, len, for_write);
} }
// NewSequentialFile opens a file for sequential reading.
IOStatus NewSequentialFile(const std::string& fname, IOStatus NewSequentialFile(const std::string& fname,
const FileOptions& options, const FileOptions& options,
std::unique_ptr<FSSequentialFile>* result, std::unique_ptr<FSSequentialFile>* result,
@ -716,7 +614,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return status; return status;
} }
// NewRandomAccessFile opens a file for random read access.
IOStatus NewRandomAccessFile(const std::string& fname, IOStatus NewRandomAccessFile(const std::string& fname,
const FileOptions& options, const FileOptions& options,
std::unique_ptr<FSRandomAccessFile>* result, std::unique_ptr<FSRandomAccessFile>* result,
@ -747,7 +644,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return status; return status;
} }
// NewWritableFile opens a file for sequential writing.
IOStatus NewWritableFile(const std::string& fname, const FileOptions& options, IOStatus NewWritableFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<FSWritableFile>* result, std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override { IODebugContext* dbg) override {
@ -765,13 +661,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return CreateWritableEncryptedFile(fname, underlying, options, result, dbg); return CreateWritableEncryptedFile(fname, underlying, options, result, dbg);
} }
// Create an object that writes to a new file with the specified
// name. Deletes any existing file with the same name and creates a
// new file. On success, stores a pointer to the new file in
// *result and returns OK. On failure stores nullptr in *result and
// returns non-OK.
//
// The returned file will only be accessed by one thread at a time.
IOStatus ReopenWritableFile(const std::string& fname, IOStatus ReopenWritableFile(const std::string& fname,
const FileOptions& options, const FileOptions& options,
std::unique_ptr<FSWritableFile>* result, std::unique_ptr<FSWritableFile>* result,
@ -790,7 +679,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return CreateWritableEncryptedFile(fname, underlying, options, result, dbg); return CreateWritableEncryptedFile(fname, underlying, options, result, dbg);
} }
// Reuse an existing file by renaming it and opening it as writable.
IOStatus ReuseWritableFile(const std::string& fname, IOStatus ReuseWritableFile(const std::string& fname,
const std::string& old_fname, const std::string& old_fname,
const FileOptions& options, const FileOptions& options,
@ -810,11 +698,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return CreateWritableEncryptedFile(fname, underlying, options, result, dbg); return CreateWritableEncryptedFile(fname, underlying, options, result, dbg);
} }
// Open `fname` for random read and write, if file doesn't exist the file
// will be created. On success, stores a pointer to the new file in
// *result and returns OK. On failure returns non-OK.
//
// The returned file will only be accessed by one thread at a time.
IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options, IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options,
std::unique_ptr<FSRandomRWFile>* result, std::unique_ptr<FSRandomRWFile>* result,
IODebugContext* dbg) override { IODebugContext* dbg) override {
@ -854,20 +737,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return status; return status;
} }
// Store in *result the attributes of the children of the specified
// directory.
// In case the implementation lists the directory prior to iterating the
// files
// and files are concurrently deleted, the deleted files will be omitted
// from
// result.
// The name attributes are relative to "dir".
// Original contents of *results are dropped.
// Returns OK if "dir" exists and "*result" contains its children.
// NotFound if "dir" does not exist, the calling process does not
// have
// permission to access "dir", or if "dir" is invalid.
// IOError if an IO Error was encountered
IOStatus GetChildrenFileAttributes(const std::string& dir, IOStatus GetChildrenFileAttributes(const std::string& dir,
const IOOptions& options, const IOOptions& options,
std::vector<FileAttributes>* result, std::vector<FileAttributes>* result,
@ -894,7 +763,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return IOStatus::OK(); return IOStatus::OK();
} }
// Store the size of fname in *file_size.
IOStatus GetFileSize(const std::string& fname, const IOOptions& options, IOStatus GetFileSize(const std::string& fname, const IOOptions& options,
uint64_t* file_size, IODebugContext* dbg) override { uint64_t* file_size, IODebugContext* dbg) override {
auto status = auto status =
@ -940,16 +808,13 @@ std::shared_ptr<FileSystem> NewEncryptedFS(
return nullptr; return nullptr;
} }
} }
// Returns an Env that encrypts data when stored on disk and decrypts data when
// read from disk.
Env* NewEncryptedEnv(Env* base_env, Env* NewEncryptedEnv(Env* base_env,
const std::shared_ptr<EncryptionProvider>& provider) { const std::shared_ptr<EncryptionProvider>& provider) {
return new CompositeEnvWrapper( return new CompositeEnvWrapper(
base_env, NewEncryptedFS(base_env->GetFileSystem(), provider)); base_env, NewEncryptedFS(base_env->GetFileSystem(), provider));
} }
// Encrypt one or more (partial) blocks of data at the file offset.
// Length of data is given in dataSize.
Status BlockAccessCipherStream::Encrypt(uint64_t fileOffset, char* data, Status BlockAccessCipherStream::Encrypt(uint64_t fileOffset, char* data,
size_t dataSize) { size_t dataSize) {
// Calculate block index // Calculate block index
@ -968,7 +833,7 @@ Status BlockAccessCipherStream::Encrypt(uint64_t fileOffset, char* data,
if (n != blockSize) { if (n != blockSize) {
// We're not encrypting a full block. // We're not encrypting a full block.
// Copy data to blockBuffer // Copy data to blockBuffer
if (!blockBuffer.get()) { if (!blockBuffer) {
// Allocate buffer // Allocate buffer
blockBuffer = std::unique_ptr<char[]>(new char[blockSize]); blockBuffer = std::unique_ptr<char[]>(new char[blockSize]);
} }
@ -994,8 +859,6 @@ Status BlockAccessCipherStream::Encrypt(uint64_t fileOffset, char* data,
} }
} }
// Decrypt one or more (partial) blocks of data at the file offset.
// Length of data is given in dataSize.
Status BlockAccessCipherStream::Decrypt(uint64_t fileOffset, char* data, Status BlockAccessCipherStream::Decrypt(uint64_t fileOffset, char* data,
size_t dataSize) { size_t dataSize) {
// Calculate block index // Calculate block index
@ -1014,7 +877,7 @@ Status BlockAccessCipherStream::Decrypt(uint64_t fileOffset, char* data,
if (n != blockSize) { if (n != blockSize) {
// We're not decrypting a full block. // We're not decrypting a full block.
// Copy data to blockBuffer // Copy data to blockBuffer
if (!blockBuffer.get()) { if (!blockBuffer) {
// Allocate buffer // Allocate buffer
blockBuffer = std::unique_ptr<char[]>(new char[blockSize]); blockBuffer = std::unique_ptr<char[]>(new char[blockSize]);
} }
@ -1055,6 +918,7 @@ static std::unordered_map<std::string, OptionTypeInfo>
{0 /* No offset, whole struct*/, OptionType::kInt, {0 /* No offset, whole struct*/, OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
}; };
// Implements a BlockCipher using ROT13. // Implements a BlockCipher using ROT13.
// //
// Note: This is a sample implementation of BlockCipher, // Note: This is a sample implementation of BlockCipher,
@ -1071,22 +935,17 @@ class ROT13BlockCipher : public BlockCipher {
static const char* kClassName() { return "ROT13"; } static const char* kClassName() { return "ROT13"; }
const char* Name() const override { return kClassName(); } const char* Name() const override { return kClassName(); }
// BlockSize returns the size of each block supported by this cipher stream.
size_t BlockSize() override { return blockSize_; }
// Encrypt a block of data. size_t BlockSize() override { return blockSize_; }
// Length of data is equal to BlockSize().
Status Encrypt(char* data) override { Status Encrypt(char* data) override {
for (size_t i = 0; i < blockSize_; ++i) { for (size_t i = 0; i < blockSize_; ++i) {
data[i] += 13; data[i] += 13;
} }
return Status::OK(); return Status::OK();
} }
// Decrypt a block of data.
// Length of data is equal to BlockSize().
Status Decrypt(char* data) override { return Encrypt(data); } Status Decrypt(char* data) override { return Encrypt(data); }
}; };
static const std::unordered_map<std::string, OptionTypeInfo> static const std::unordered_map<std::string, OptionTypeInfo>
ctr_encryption_provider_type_info = { ctr_encryption_provider_type_info = {
{"cipher", {"cipher",
@ -1096,14 +955,11 @@ static const std::unordered_map<std::string, OptionTypeInfo>
}; };
} // anonymous namespace } // anonymous namespace
// Allocate scratch space which is passed to EncryptBlock/DecryptBlock.
void CTRCipherStream::AllocateScratch(std::string& scratch) { void CTRCipherStream::AllocateScratch(std::string& scratch) {
auto blockSize = cipher_->BlockSize(); auto blockSize = cipher_->BlockSize();
scratch.reserve(blockSize); scratch.reserve(blockSize);
} }
// Encrypt a block of data at the given block index.
// Length of data is equal to BlockSize();
Status CTRCipherStream::EncryptBlock(uint64_t blockIndex, char* data, Status CTRCipherStream::EncryptBlock(uint64_t blockIndex, char* data,
char* scratch) { char* scratch) {
// Create nonce + counter // Create nonce + counter
@ -1111,7 +967,7 @@ Status CTRCipherStream::EncryptBlock(uint64_t blockIndex, char* data,
memmove(scratch, iv_.data(), blockSize); memmove(scratch, iv_.data(), blockSize);
EncodeFixed64(scratch, blockIndex + initialCounter_); EncodeFixed64(scratch, blockIndex + initialCounter_);
// Encrypt nonce+counter // Encrypt nonce + counter
auto status = cipher_->Encrypt(scratch); auto status = cipher_->Encrypt(scratch);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
@ -1124,8 +980,6 @@ Status CTRCipherStream::EncryptBlock(uint64_t blockIndex, char* data,
return Status::OK(); return Status::OK();
} }
// Decrypt a block of data at the given block index.
// Length of data is equal to BlockSize();
Status CTRCipherStream::DecryptBlock(uint64_t blockIndex, char* data, Status CTRCipherStream::DecryptBlock(uint64_t blockIndex, char* data,
char* scratch) { char* scratch) {
// For CTR decryption & encryption are the same // For CTR decryption & encryption are the same
@ -1147,10 +1001,6 @@ bool CTREncryptionProvider::IsInstanceOf(const std::string& name) const {
} }
} }
// GetPrefixLength returns the length of the prefix that is added to every file
// and used for storing encryption options.
// For optimal performance, the prefix length should be a multiple of
// the page size.
size_t CTREncryptionProvider::GetPrefixLength() const { size_t CTREncryptionProvider::GetPrefixLength() const {
return defaultPrefixLength; return defaultPrefixLength;
} }
@ -1179,8 +1029,6 @@ static void decodeCTRParameters(const char* prefix, size_t blockSize,
iv = Slice(prefix + blockSize, blockSize); iv = Slice(prefix + blockSize, blockSize);
} }
// CreateNewPrefix initialized an allocated block of prefix memory
// for a new file.
Status CTREncryptionProvider::CreateNewPrefix(const std::string& /*fname*/, Status CTREncryptionProvider::CreateNewPrefix(const std::string& /*fname*/,
char* prefix, char* prefix,
size_t prefixLength) const { size_t prefixLength) const {
@ -1212,10 +1060,8 @@ Status CTREncryptionProvider::CreateNewPrefix(const std::string& /*fname*/,
status = cipherStream.Encrypt(0, prefix + (2 * blockSize), status = cipherStream.Encrypt(0, prefix + (2 * blockSize),
prefixLength - (2 * blockSize)); prefixLength - (2 * blockSize));
} }
if (!status.ok()) {
return status; return status;
}
return Status::OK();
} }
// PopulateSecretPrefixPart initializes the data into a new prefix block // PopulateSecretPrefixPart initializes the data into a new prefix block
@ -1267,7 +1113,7 @@ Status CTREncryptionProvider::CreateCipherStream(
} }
// CreateCipherStreamFromPrefix creates a block access cipher stream for a file // CreateCipherStreamFromPrefix creates a block access cipher stream for a file
// given given name and options. The given prefix is already decrypted. // given name and options. The given prefix is already decrypted.
Status CTREncryptionProvider::CreateCipherStreamFromPrefix( Status CTREncryptionProvider::CreateCipherStreamFromPrefix(
const std::string& /*fname*/, const EnvOptions& /*options*/, const std::string& /*fname*/, const EnvOptions& /*options*/,
uint64_t initialCounter, const Slice& iv, const Slice& /*prefix*/, uint64_t initialCounter, const Slice& iv, const Slice& /*prefix*/,

17
env/env_encryption_ctr.h vendored
Unescape View File

@ -27,19 +27,13 @@ class CTRCipherStream final : public BlockAccessCipherStream {
: cipher_(c), iv_(iv, c->BlockSize()), initialCounter_(initialCounter){}; : cipher_(c), iv_(iv, c->BlockSize()), initialCounter_(initialCounter){};
virtual ~CTRCipherStream(){}; virtual ~CTRCipherStream(){};
// BlockSize returns the size of each block supported by this cipher stream.
size_t BlockSize() override { return cipher_->BlockSize(); } size_t BlockSize() override { return cipher_->BlockSize(); }
protected: protected:
// Allocate scratch space which is passed to EncryptBlock/DecryptBlock.
void AllocateScratch(std::string&) override; void AllocateScratch(std::string&) override;
// Encrypt a block of data at the given block index.
// Length of data is equal to BlockSize();
Status EncryptBlock(uint64_t blockIndex, char* data, char* scratch) override; Status EncryptBlock(uint64_t blockIndex, char* data, char* scratch) override;
// Decrypt a block of data at the given block index.
// Length of data is equal to BlockSize();
Status DecryptBlock(uint64_t blockIndex, char* data, char* scratch) override; Status DecryptBlock(uint64_t blockIndex, char* data, char* scratch) override;
}; };
@ -66,20 +60,9 @@ class CTREncryptionProvider : public EncryptionProvider {
static const char* kClassName() { return "CTR"; } static const char* kClassName() { return "CTR"; }
const char* Name() const override { return kClassName(); } const char* Name() const override { return kClassName(); }
bool IsInstanceOf(const std::string& name) const override; bool IsInstanceOf(const std::string& name) const override;
// GetPrefixLength returns the length of the prefix that is added to every
// file
// and used for storing encryption options.
// For optimal performance when using direct IO, the prefix length should be a
// multiple of the page size.
size_t GetPrefixLength() const override; size_t GetPrefixLength() const override;
// CreateNewPrefix initialized an allocated block of prefix memory
// for a new file.
Status CreateNewPrefix(const std::string& fname, char* prefix, Status CreateNewPrefix(const std::string& fname, char* prefix,
size_t prefixLength) const override; size_t prefixLength) const override;
// CreateCipherStream creates a block access cipher stream for a file given
// given name and options.
Status CreateCipherStream( Status CreateCipherStream(
const std::string& fname, const EnvOptions& options, Slice& prefix, const std::string& fname, const EnvOptions& options, Slice& prefix,
std::unique_ptr<BlockAccessCipherStream>* result) override; std::unique_ptr<BlockAccessCipherStream>* result) override;

118
include/rocksdb/env_encryption.h
Unescape View File

@ -67,7 +67,7 @@ class BlockAccessCipherStream {
// including data loss, unreported corruption, deadlocks, and more. // including data loss, unreported corruption, deadlocks, and more.
class BlockCipher : public Customizable { class BlockCipher : public Customizable {
public: public:
virtual ~BlockCipher(){}; virtual ~BlockCipher() {}
// Creates a new BlockCipher from the input config_options and value // Creates a new BlockCipher from the input config_options and value
// The value describes the type of provider (and potentially optional // The value describes the type of provider (and potentially optional
@ -114,13 +114,13 @@ class BlockCipher : public Customizable {
// including data loss, unreported corruption, deadlocks, and more. // including data loss, unreported corruption, deadlocks, and more.
class EncryptionProvider : public Customizable { class EncryptionProvider : public Customizable {
public: public:
virtual ~EncryptionProvider(){}; virtual ~EncryptionProvider() {}
// Creates a new EncryptionProvider from the input config_options and value // Creates a new EncryptionProvider from the input config_options and value.
// The value describes the type of provider (and potentially optional // The value describes the type of provider (and potentially optional
// configuration parameters) used to create this provider. // configuration parameters) used to create this provider.
// For example, if the value is "CTR", a CTREncryptionProvider will be // For example, if the value is "CTR", a CTREncryptionProvider will be
// created. If the value is ends with "://test" (e.g CTR://test"), the // created. If the value is end with "://test" (e.g CTR://test"), the
// provider will be initialized in "TEST" mode prior to being returned. // provider will be initialized in "TEST" mode prior to being returned.
// //
// @param config_options Options to control how this provider is created // @param config_options Options to control how this provider is created
@ -153,7 +153,7 @@ class EncryptionProvider : public Customizable {
size_t prefixLength) const = 0; size_t prefixLength) const = 0;
// Method to add a new cipher key for use by the EncryptionProvider. // Method to add a new cipher key for use by the EncryptionProvider.
// @param description Descriptor for this key. // @param descriptor Descriptor for this key
// @param cipher The cryptographic key to use // @param cipher The cryptographic key to use
// @param len The length of the cipher key // @param len The length of the cipher key
// @param for_write If true, this cipher should be used for writing files. // @param for_write If true, this cipher should be used for writing files.
@ -165,15 +165,15 @@ class EncryptionProvider : public Customizable {
size_t len, bool for_write) = 0; size_t len, bool for_write) = 0;
// CreateCipherStream creates a block access cipher stream for a file given // CreateCipherStream creates a block access cipher stream for a file given
// given name and options. // name and options.
virtual Status CreateCipherStream( virtual Status CreateCipherStream(
const std::string& fname, const EnvOptions& options, Slice& prefix, const std::string& fname, const EnvOptions& options, Slice& prefix,
std::unique_ptr<BlockAccessCipherStream>* result) = 0; std::unique_ptr<BlockAccessCipherStream>* result) = 0;
// Returns a string representing an encryption marker prefix for this // Returns a string representing an encryption marker prefix for this
// provider. If a marker is provided, this marker can be used to tell whether // provider. If a marker is provided, this marker can be used to tell whether
// or not a file is encrypted by this provider. The maker will also be part // a file is encrypted by this provider. The marker will also be part of any
// of any encryption prefix for this provider. // encryption prefix for this provider.
virtual std::string GetMarker() const { return ""; } virtual std::string GetMarker() const { return ""; }
}; };
@ -182,7 +182,7 @@ class EncryptedSequentialFile : public FSSequentialFile {
std::unique_ptr<FSSequentialFile> file_; std::unique_ptr<FSSequentialFile> file_;
std::unique_ptr<BlockAccessCipherStream> stream_; std::unique_ptr<BlockAccessCipherStream> stream_;
uint64_t offset_; uint64_t offset_;
size_t prefixLength_; const size_t prefixLength_;
public: public:
// Default ctor. Given underlying sequential file is supposed to be at // Default ctor. Given underlying sequential file is supposed to be at
@ -195,47 +195,22 @@ class EncryptedSequentialFile : public FSSequentialFile {
offset_(prefixLength), offset_(prefixLength),
prefixLength_(prefixLength) {} prefixLength_(prefixLength) {}
// Read up to "n" bytes from the file. "scratch[0..n-1]" may be
// written by this routine. Sets "*result" to the data that was
// read (including if fewer than "n" bytes were successfully read).
// May set "*result" to point at data in "scratch[0..n-1]", so
// "scratch[0..n-1]" must be live when "*result" is used.
// If an error was encountered, returns a non-OK status.
//
// REQUIRES: External synchronization
IOStatus Read(size_t n, const IOOptions& options, Slice* result, IOStatus Read(size_t n, const IOOptions& options, Slice* result,
char* scratch, IODebugContext* dbg) override; char* scratch, IODebugContext* dbg) override;
// Skip "n" bytes from the file. This is guaranteed to be no
// slower that reading the same data, but may be faster.
//
// If end of file is reached, skipping will stop at the end of the
// file, and Skip will return OK.
//
// REQUIRES: External synchronization
IOStatus Skip(uint64_t n) override; IOStatus Skip(uint64_t n) override;
// Indicates the upper layers if the current SequentialFile implementation
// uses direct IO.
bool use_direct_io() const override; bool use_direct_io() const override;
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t GetRequiredBufferAlignment() const override; size_t GetRequiredBufferAlignment() const override;
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
IOStatus InvalidateCache(size_t offset, size_t length) override; IOStatus InvalidateCache(size_t offset, size_t length) override;
// Positioned Read for direct I/O
// If Direct I/O enabled, offset, n, and scratch should be properly aligned
IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& options, IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& options,
Slice* result, char* scratch, Slice* result, char* scratch,
IODebugContext* dbg) override; IODebugContext* dbg) override;
}; };
// A file abstraction for randomly reading the contents of a file.
class EncryptedRandomAccessFile : public FSRandomAccessFile { class EncryptedRandomAccessFile : public FSRandomAccessFile {
protected: protected:
std::unique_ptr<FSRandomAccessFile> file_; std::unique_ptr<FSRandomAccessFile> file_;
@ -250,60 +225,24 @@ class EncryptedRandomAccessFile : public FSRandomAccessFile {
stream_(std::move(s)), stream_(std::move(s)),
prefixLength_(prefixLength) {} prefixLength_(prefixLength) {}
// Read up to "n" bytes from the file starting at "offset".
// "scratch[0..n-1]" may be written by this routine. Sets "*result"
// to the data that was read (including if fewer than "n" bytes were
// successfully read). May set "*result" to point at data in
// "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
// "*result" is used. If an error was encountered, returns a non-OK
// status.
//
// Safe for concurrent use by multiple threads.
// If Direct I/O enabled, offset, n, and scratch should be aligned properly.
IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
Slice* result, char* scratch, Slice* result, char* scratch,
IODebugContext* dbg) const override; IODebugContext* dbg) const override;
// Readahead the file starting from offset by n bytes for caching.
IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options, IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options,
IODebugContext* dbg) override; IODebugContext* dbg) override;
// Tries to get an unique ID for this file that will be the same each time
// the file is opened (and will stay the same while the file is open).
// Furthermore, it tries to make this ID at most "max_size" bytes. If such an
// ID can be created this function returns the length of the ID and places it
// in "id"; otherwise, this function returns 0, in which case "id"
// may not have been modified.
//
// This function guarantees, for IDs from a given environment, two unique ids
// cannot be made equal to each other by adding arbitrary bytes to one of
// them. That is, no unique ID is the prefix of another.
//
// This function guarantees that the returned ID will not be interpretable as
// a single varint.
//
// Note: these IDs are only valid for the duration of the process.
size_t GetUniqueId(char* id, size_t max_size) const override; size_t GetUniqueId(char* id, size_t max_size) const override;
void Hint(AccessPattern pattern) override; void Hint(AccessPattern pattern) override;
// Indicates the upper layers if the current RandomAccessFile implementation
// uses direct IO.
bool use_direct_io() const override; bool use_direct_io() const override;
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t GetRequiredBufferAlignment() const override; size_t GetRequiredBufferAlignment() const override;
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
IOStatus InvalidateCache(size_t offset, size_t length) override; IOStatus InvalidateCache(size_t offset, size_t length) override;
}; };
// A file abstraction for sequential writing. The implementation
// must provide buffering since callers may append small fragments
// at a time to the file.
class EncryptedWritableFile : public FSWritableFile { class EncryptedWritableFile : public FSWritableFile {
protected: protected:
std::unique_ptr<FSWritableFile> file_; std::unique_ptr<FSWritableFile> file_;
@ -328,50 +267,22 @@ class EncryptedWritableFile : public FSWritableFile {
const IOOptions& options, const IOOptions& options,
IODebugContext* dbg) override; IODebugContext* dbg) override;
// true if Sync() and Fsync() are safe to call concurrently with Append()
// and Flush().
bool IsSyncThreadSafe() const override; bool IsSyncThreadSafe() const override;
// Indicates the upper layers if the current WritableFile implementation
// uses direct IO.
bool use_direct_io() const override; bool use_direct_io() const override;
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t GetRequiredBufferAlignment() const override; size_t GetRequiredBufferAlignment() const override;
/*
* Get the size of valid data in the file.
*/
uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) override; uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) override;
// Truncate is necessary to trim the file to the correct size
// before closing. It is not always possible to keep track of the file
// size due to whole pages writes. The behavior is undefined if called
// with other writes to follow.
IOStatus Truncate(uint64_t size, const IOOptions& options, IOStatus Truncate(uint64_t size, const IOOptions& options,
IODebugContext* dbg) override; IODebugContext* dbg) override;
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
// This call has no effect on dirty pages in the cache.
IOStatus InvalidateCache(size_t offset, size_t length) override; IOStatus InvalidateCache(size_t offset, size_t length) override;
// Sync a file range with disk.
// offset is the starting byte of the file range to be synchronized.
// nbytes specifies the length of the range to be synchronized.
// This asks the OS to initiate flushing the cached data to disk,
// without waiting for completion.
// Default implementation does nothing.
IOStatus RangeSync(uint64_t offset, uint64_t nbytes, const IOOptions& options, IOStatus RangeSync(uint64_t offset, uint64_t nbytes, const IOOptions& options,
IODebugContext* dbg) override; IODebugContext* dbg) override;
// PrepareWrite performs any necessary preparation for a write
// before the write actually occurs. This allows for pre-allocation
// of space on devices where it can result in less file
// fragmentation and/or less waste from over-zealous filesystem
// pre-allocation.
void PrepareWrite(size_t offset, size_t len, const IOOptions& options, void PrepareWrite(size_t offset, size_t len, const IOOptions& options,
IODebugContext* dbg) override; IODebugContext* dbg) override;
@ -380,7 +291,6 @@ class EncryptedWritableFile : public FSWritableFile {
void GetPreallocationStatus(size_t* block_size, void GetPreallocationStatus(size_t* block_size,
size_t* last_allocated_block) override; size_t* last_allocated_block) override;
// Pre-allocates space for a file.
IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& options, IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& options,
IODebugContext* dbg) override; IODebugContext* dbg) override;
@ -391,7 +301,6 @@ class EncryptedWritableFile : public FSWritableFile {
IOStatus Close(const IOOptions& options, IODebugContext* dbg) override; IOStatus Close(const IOOptions& options, IODebugContext* dbg) override;
}; };
// A file abstraction for random reading and writing.
class EncryptedRandomRWFile : public FSRandomRWFile { class EncryptedRandomRWFile : public FSRandomRWFile {
protected: protected:
std::unique_ptr<FSRandomRWFile> file_; std::unique_ptr<FSRandomRWFile> file_;
@ -406,22 +315,13 @@ class EncryptedRandomRWFile : public FSRandomRWFile {
stream_(std::move(s)), stream_(std::move(s)),
prefixLength_(prefixLength) {} prefixLength_(prefixLength) {}
// Indicates if the class makes use of direct I/O
// If false you must pass aligned buffer to Write()
bool use_direct_io() const override; bool use_direct_io() const override;
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t GetRequiredBufferAlignment() const override; size_t GetRequiredBufferAlignment() const override;
// Write bytes in `data` at offset `offset`, Returns Status::OK() on success.
// Pass aligned buffer when use_direct_io() returns true.
IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options, IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options,
IODebugContext* dbg) override; IODebugContext* dbg) override;
// Read up to `n` bytes starting from offset `offset` and store them in
// result, provided `scratch` size should be at least `n`.
// Returns Status::OK() on success.
IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
Slice* result, char* scratch, Slice* result, char* scratch,
IODebugContext* dbg) const override; IODebugContext* dbg) const override;

Write Preview
Loading…
Cancel
Save