@ -30,14 +30,6 @@ std::shared_ptr<EncryptionProvider> EncryptionProvider::NewCTRProvider(
return std : : make_shared < CTREncryptionProvider > ( cipher ) ;
}
// Read up to "n" bytes from the file. "scratch[0..n-1]" may be
// written by this routine. Sets "*result" to the data that was
// read (including if fewer than "n" bytes were successfully read).
// May set "*result" to point at data in "scratch[0..n-1]", so
// "scratch[0..n-1]" must be live when "*result" is used.
// If an error was encountered, returns a non-OK status.
//
// REQUIRES: External synchronization
IOStatus EncryptedSequentialFile : : Read ( size_t n , const IOOptions & options ,
Slice * result , char * scratch ,
IODebugContext * dbg ) {
@ -52,19 +44,12 @@ IOStatus EncryptedSequentialFile::Read(size_t n, const IOOptions& options,
stream_ - > Decrypt ( offset_ , ( char * ) result - > data ( ) , result - > size ( ) ) ) ;
}
if ( io_s . ok ( ) ) {
offset_ + = result - > size ( ) ; // We've already ready data from disk, so update
offset_ + = result - > size ( ) ; // We've already read data from disk, so update
// offset_ even if decryption fails.
}
return io_s ;
}
// Skip "n" bytes from the file. This is guaranteed to be no
// slower that reading the same data, but may be faster.
//
// If end of file is reached, skipping will stop at the end of the
// file, and Skip will return OK.
//
// REQUIRES: External synchronization
IOStatus EncryptedSequentialFile : : Skip ( uint64_t n ) {
auto status = file_ - > Skip ( n ) ;
if ( ! status . ok ( ) ) {
@ -74,28 +59,19 @@ IOStatus EncryptedSequentialFile::Skip(uint64_t n) {
return status ;
}
// Indicates the upper layers if the current SequentialFile implementation
// uses direct IO.
bool EncryptedSequentialFile : : use_direct_io ( ) const {
return file_ - > use_direct_io ( ) ;
}
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t EncryptedSequentialFile : : GetRequiredBufferAlignment ( ) const {
return file_ - > GetRequiredBufferAlignment ( ) ;
}
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
IOStatus EncryptedSequentialFile : : InvalidateCache ( size_t offset ,
size_t length ) {
return file_ - > InvalidateCache ( offset + prefixLength_ , length ) ;
}
// Positioned Read for direct I/O
// If Direct I/O enabled, offset, n, and scratch should be properly aligned
IOStatus EncryptedSequentialFile : : PositionedRead ( uint64_t offset , size_t n ,
const IOOptions & options ,
Slice * result , char * scratch ,
@ -115,16 +91,6 @@ IOStatus EncryptedSequentialFile::PositionedRead(uint64_t offset, size_t n,
return io_s ;
}
// Read up to "n" bytes from the file starting at "offset".
// "scratch[0..n-1]" may be written by this routine. Sets "*result"
// to the data that was read (including if fewer than "n" bytes were
// successfully read). May set "*result" to point at data in
// "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
// "*result" is used. If an error was encountered, returns a non-OK
// status.
//
// Safe for concurrent use by multiple threads.
// If Direct I/O enabled, offset, n, and scratch should be aligned properly.
IOStatus EncryptedRandomAccessFile : : Read ( uint64_t offset , size_t n ,
const IOOptions & options ,
Slice * result , char * scratch ,
@ -143,29 +109,12 @@ IOStatus EncryptedRandomAccessFile::Read(uint64_t offset, size_t n,
return io_s ;
}
// Readahead the file starting from offset by n bytes for caching.
IOStatus EncryptedRandomAccessFile : : Prefetch ( uint64_t offset , size_t n ,
const IOOptions & options ,
IODebugContext * dbg ) {
// return Status::OK();
return file_ - > Prefetch ( offset + prefixLength_ , n , options , dbg ) ;
}
// Tries to get an unique ID for this file that will be the same each time
// the file is opened (and will stay the same while the file is open).
// Furthermore, it tries to make this ID at most "max_size" bytes. If such an
// ID can be created this function returns the length of the ID and places it
// in "id"; otherwise, this function returns 0, in which case "id"
// may not have been modified.
//
// This function guarantees, for IDs from a given environment, two unique ids
// cannot be made equal to each other by adding arbitrary bytes to one of
// them. That is, no unique ID is the prefix of another.
//
// This function guarantees that the returned ID will not be interpretable as
// a single varint.
//
// Note: these IDs are only valid for the duration of the process.
size_t EncryptedRandomAccessFile : : GetUniqueId ( char * id , size_t max_size ) const {
return file_ - > GetUniqueId ( id , max_size ) ;
} ;
@ -174,29 +123,19 @@ void EncryptedRandomAccessFile::Hint(AccessPattern pattern) {
file_ - > Hint ( pattern ) ;
}
// Indicates the upper layers if the current RandomAccessFile implementation
// uses direct IO.
bool EncryptedRandomAccessFile : : use_direct_io ( ) const {
return file_ - > use_direct_io ( ) ;
}
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t EncryptedRandomAccessFile : : GetRequiredBufferAlignment ( ) const {
return file_ - > GetRequiredBufferAlignment ( ) ;
}
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
IOStatus EncryptedRandomAccessFile : : InvalidateCache ( size_t offset ,
size_t length ) {
return file_ - > InvalidateCache ( offset + prefixLength_ , length ) ;
}
// A file abstraction for sequential writing. The implementation
// must provide buffering since callers may append small fragments
// at a time to the file.
IOStatus EncryptedWritableFile : : Append ( const Slice & data ,
const IOOptions & options ,
IODebugContext * dbg ) {
@ -252,67 +191,39 @@ IOStatus EncryptedWritableFile::PositionedAppend(const Slice& data,
return file_ - > PositionedAppend ( dataToAppend , offset , options , dbg ) ;
}
// Indicates the upper layers if the current WritableFile implementation
// uses direct IO.
bool EncryptedWritableFile : : use_direct_io ( ) const {
return file_ - > use_direct_io ( ) ;
}
// true if Sync() and Fsync() are safe to call concurrently with Append()
// and Flush().
bool EncryptedWritableFile : : IsSyncThreadSafe ( ) const {
return file_ - > IsSyncThreadSafe ( ) ;
}
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t EncryptedWritableFile : : GetRequiredBufferAlignment ( ) const {
return file_ - > GetRequiredBufferAlignment ( ) ;
}
/*
* Get the size of valid data in the file .
*/
uint64_t EncryptedWritableFile : : GetFileSize ( const IOOptions & options ,
IODebugContext * dbg ) {
return file_ - > GetFileSize ( options , dbg ) - prefixLength_ ;
}
// Truncate is necessary to trim the file to the correct size
// before closing. It is not always possible to keep track of the file
// size due to whole pages writes. The behavior is undefined if called
// with other writes to follow.
IOStatus EncryptedWritableFile : : Truncate ( uint64_t size ,
const IOOptions & options ,
IODebugContext * dbg ) {
return file_ - > Truncate ( size + prefixLength_ , options , dbg ) ;
}
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
// This call has no effect on dirty pages in the cache.
IOStatus EncryptedWritableFile : : InvalidateCache ( size_t offset , size_t length ) {
return file_ - > InvalidateCache ( offset + prefixLength_ , length ) ;
}
// Sync a file range with disk.
// offset is the starting byte of the file range to be synchronized.
// nbytes specifies the length of the range to be synchronized.
// This asks the OS to initiate flushing the cached data to disk,
// without waiting for completion.
// Default implementation does nothing.
IOStatus EncryptedWritableFile : : RangeSync ( uint64_t offset , uint64_t nbytes ,
const IOOptions & options ,
IODebugContext * dbg ) {
return file_ - > RangeSync ( offset + prefixLength_ , nbytes , options , dbg ) ;
}
// PrepareWrite performs any necessary preparation for a write
// before the write actually occurs. This allows for pre-allocation
// of space on devices where it can result in less file
// fragmentation and/or less waste from over-zealous filesystem
// pre-allocation.
void EncryptedWritableFile : : PrepareWrite ( size_t offset , size_t len ,
const IOOptions & options ,
IODebugContext * dbg ) {
@ -330,7 +241,6 @@ void EncryptedWritableFile::GetPreallocationStatus(
file_ - > GetPreallocationStatus ( block_size , last_allocated_block ) ;
}
// Pre-allocates space for a file.
IOStatus EncryptedWritableFile : : Allocate ( uint64_t offset , uint64_t len ,
const IOOptions & options ,
IODebugContext * dbg ) {
@ -352,22 +262,14 @@ IOStatus EncryptedWritableFile::Close(const IOOptions& options,
return file_ - > Close ( options , dbg ) ;
}
// A file abstraction for random reading and writing.
// Indicates if the class makes use of direct I/O
// If false you must pass aligned buffer to Write()
bool EncryptedRandomRWFile : : use_direct_io ( ) const {
return file_ - > use_direct_io ( ) ;
}
// Use the returned alignment value to allocate
// aligned buffer for Direct I/O
size_t EncryptedRandomRWFile : : GetRequiredBufferAlignment ( ) const {
return file_ - > GetRequiredBufferAlignment ( ) ;
}
// Write bytes in `data` at offset `offset`, Returns Status::OK() on success.
// Pass aligned buffer when use_direct_io() returns true.
IOStatus EncryptedRandomRWFile : : Write ( uint64_t offset , const Slice & data ,
const IOOptions & options ,
IODebugContext * dbg ) {
@ -394,9 +296,6 @@ IOStatus EncryptedRandomRWFile::Write(uint64_t offset, const Slice& data,
return file_ - > Write ( offset , dataToWrite , options , dbg ) ;
}
// Read up to `n` bytes starting from offset `offset` and store them in
// result, provided `scratch` size should be at least `n`.
// Returns Status::OK() on success.
IOStatus EncryptedRandomRWFile : : Read ( uint64_t offset , size_t n ,
const IOOptions & options , Slice * result ,
char * scratch , IODebugContext * dbg ) const {
@ -678,7 +577,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return provider_ - > AddCipher ( descriptor , cipher , len , for_write ) ;
}
// NewSequentialFile opens a file for sequential reading.
IOStatus NewSequentialFile ( const std : : string & fname ,
const FileOptions & options ,
std : : unique_ptr < FSSequentialFile > * result ,
@ -716,7 +614,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return status ;
}
// NewRandomAccessFile opens a file for random read access.
IOStatus NewRandomAccessFile ( const std : : string & fname ,
const FileOptions & options ,
std : : unique_ptr < FSRandomAccessFile > * result ,
@ -747,7 +644,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return status ;
}
// NewWritableFile opens a file for sequential writing.
IOStatus NewWritableFile ( const std : : string & fname , const FileOptions & options ,
std : : unique_ptr < FSWritableFile > * result ,
IODebugContext * dbg ) override {
@ -765,13 +661,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return CreateWritableEncryptedFile ( fname , underlying , options , result , dbg ) ;
}
// Create an object that writes to a new file with the specified
// name. Deletes any existing file with the same name and creates a
// new file. On success, stores a pointer to the new file in
// *result and returns OK. On failure stores nullptr in *result and
// returns non-OK.
//
// The returned file will only be accessed by one thread at a time.
IOStatus ReopenWritableFile ( const std : : string & fname ,
const FileOptions & options ,
std : : unique_ptr < FSWritableFile > * result ,
@ -790,7 +679,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return CreateWritableEncryptedFile ( fname , underlying , options , result , dbg ) ;
}
// Reuse an existing file by renaming it and opening it as writable.
IOStatus ReuseWritableFile ( const std : : string & fname ,
const std : : string & old_fname ,
const FileOptions & options ,
@ -810,11 +698,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return CreateWritableEncryptedFile ( fname , underlying , options , result , dbg ) ;
}
// Open `fname` for random read and write, if file doesn't exist the file
// will be created. On success, stores a pointer to the new file in
// *result and returns OK. On failure returns non-OK.
//
// The returned file will only be accessed by one thread at a time.
IOStatus NewRandomRWFile ( const std : : string & fname , const FileOptions & options ,
std : : unique_ptr < FSRandomRWFile > * result ,
IODebugContext * dbg ) override {
@ -854,20 +737,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return status ;
}
// Store in *result the attributes of the children of the specified
// directory.
// In case the implementation lists the directory prior to iterating the
// files
// and files are concurrently deleted, the deleted files will be omitted
// from
// result.
// The name attributes are relative to "dir".
// Original contents of *results are dropped.
// Returns OK if "dir" exists and "*result" contains its children.
// NotFound if "dir" does not exist, the calling process does not
// have
// permission to access "dir", or if "dir" is invalid.
// IOError if an IO Error was encountered
IOStatus GetChildrenFileAttributes ( const std : : string & dir ,
const IOOptions & options ,
std : : vector < FileAttributes > * result ,
@ -894,7 +763,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
return IOStatus : : OK ( ) ;
}
// Store the size of fname in *file_size.
IOStatus GetFileSize ( const std : : string & fname , const IOOptions & options ,
uint64_t * file_size , IODebugContext * dbg ) override {
auto status =
@ -940,16 +808,13 @@ std::shared_ptr<FileSystem> NewEncryptedFS(
return nullptr ;
}
}
// Returns an Env that encrypts data when stored on disk and decrypts data when
// read from disk.
Env * NewEncryptedEnv ( Env * base_env ,
const std : : shared_ptr < EncryptionProvider > & provider ) {
return new CompositeEnvWrapper (
base_env , NewEncryptedFS ( base_env - > GetFileSystem ( ) , provider ) ) ;
}
// Encrypt one or more (partial) blocks of data at the file offset.
// Length of data is given in dataSize.
Status BlockAccessCipherStream : : Encrypt ( uint64_t fileOffset , char * data ,
size_t dataSize ) {
// Calculate block index
@ -968,7 +833,7 @@ Status BlockAccessCipherStream::Encrypt(uint64_t fileOffset, char* data,
if ( n ! = blockSize ) {
// We're not encrypting a full block.
// Copy data to blockBuffer
if ( ! blockBuffer . get ( ) ) {
if ( ! blockBuffer ) {
// Allocate buffer
blockBuffer = std : : unique_ptr < char [ ] > ( new char [ blockSize ] ) ;
}
@ -994,8 +859,6 @@ Status BlockAccessCipherStream::Encrypt(uint64_t fileOffset, char* data,
}
}
// Decrypt one or more (partial) blocks of data at the file offset.
// Length of data is given in dataSize.
Status BlockAccessCipherStream : : Decrypt ( uint64_t fileOffset , char * data ,
size_t dataSize ) {
// Calculate block index
@ -1014,7 +877,7 @@ Status BlockAccessCipherStream::Decrypt(uint64_t fileOffset, char* data,
if ( n ! = blockSize ) {
// We're not decrypting a full block.
// Copy data to blockBuffer
if ( ! blockBuffer . get ( ) ) {
if ( ! blockBuffer ) {
// Allocate buffer
blockBuffer = std : : unique_ptr < char [ ] > ( new char [ blockSize ] ) ;
}
@ -1055,6 +918,7 @@ static std::unordered_map<std::string, OptionTypeInfo>
{ 0 /* No offset, whole struct*/ , OptionType : : kInt ,
OptionVerificationType : : kNormal , OptionTypeFlags : : kNone } } ,
} ;
// Implements a BlockCipher using ROT13.
//
// Note: This is a sample implementation of BlockCipher,
@ -1071,22 +935,17 @@ class ROT13BlockCipher : public BlockCipher {
static const char * kClassName ( ) { return " ROT13 " ; }
const char * Name ( ) const override { return kClassName ( ) ; }
// BlockSize returns the size of each block supported by this cipher stream.
size_t BlockSize ( ) override { return blockSize_ ; }
// Encrypt a block of data.
// Length of data is equal to BlockSize().
size_t BlockSize ( ) override { return blockSize_ ; }
Status Encrypt ( char * data ) override {
for ( size_t i = 0 ; i < blockSize_ ; + + i ) {
data [ i ] + = 13 ;
}
return Status : : OK ( ) ;
}
// Decrypt a block of data.
// Length of data is equal to BlockSize().
Status Decrypt ( char * data ) override { return Encrypt ( data ) ; }
} ;
static const std : : unordered_map < std : : string , OptionTypeInfo >
ctr_encryption_provider_type_info = {
{ " cipher " ,
@ -1096,14 +955,11 @@ static const std::unordered_map<std::string, OptionTypeInfo>
} ;
} // anonymous namespace
// Allocate scratch space which is passed to EncryptBlock/DecryptBlock.
void CTRCipherStream : : AllocateScratch ( std : : string & scratch ) {
auto blockSize = cipher_ - > BlockSize ( ) ;
scratch . reserve ( blockSize ) ;
}
// Encrypt a block of data at the given block index.
// Length of data is equal to BlockSize();
Status CTRCipherStream : : EncryptBlock ( uint64_t blockIndex , char * data ,
char * scratch ) {
// Create nonce + counter
@ -1111,7 +967,7 @@ Status CTRCipherStream::EncryptBlock(uint64_t blockIndex, char* data,
memmove ( scratch , iv_ . data ( ) , blockSize ) ;
EncodeFixed64 ( scratch , blockIndex + initialCounter_ ) ;
// Encrypt nonce+counter
// Encrypt nonce + counter
auto status = cipher_ - > Encrypt ( scratch ) ;
if ( ! status . ok ( ) ) {
return status ;
@ -1124,8 +980,6 @@ Status CTRCipherStream::EncryptBlock(uint64_t blockIndex, char* data,
return Status : : OK ( ) ;
}
// Decrypt a block of data at the given block index.
// Length of data is equal to BlockSize();
Status CTRCipherStream : : DecryptBlock ( uint64_t blockIndex , char * data ,
char * scratch ) {
// For CTR decryption & encryption are the same
@ -1147,10 +1001,6 @@ bool CTREncryptionProvider::IsInstanceOf(const std::string& name) const {
}
}
// GetPrefixLength returns the length of the prefix that is added to every file
// and used for storing encryption options.
// For optimal performance, the prefix length should be a multiple of
// the page size.
size_t CTREncryptionProvider : : GetPrefixLength ( ) const {
return defaultPrefixLength ;
}
@ -1179,8 +1029,6 @@ static void decodeCTRParameters(const char* prefix, size_t blockSize,
iv = Slice ( prefix + blockSize , blockSize ) ;
}
// CreateNewPrefix initialized an allocated block of prefix memory
// for a new file.
Status CTREncryptionProvider : : CreateNewPrefix ( const std : : string & /*fname*/ ,
char * prefix ,
size_t prefixLength ) const {
@ -1212,10 +1060,8 @@ Status CTREncryptionProvider::CreateNewPrefix(const std::string& /*fname*/,
status = cipherStream . Encrypt ( 0 , prefix + ( 2 * blockSize ) ,
prefixLength - ( 2 * blockSize ) ) ;
}
if ( ! status . ok ( ) ) {
return status ;
}
return Status : : OK ( ) ;
return status ;
}
// PopulateSecretPrefixPart initializes the data into a new prefix block
@ -1267,7 +1113,7 @@ Status CTREncryptionProvider::CreateCipherStream(
}
// CreateCipherStreamFromPrefix creates a block access cipher stream for a file
// given given name and options. The given prefix is already decrypted.
// given name and options. The given prefix is already decrypted.
Status CTREncryptionProvider : : CreateCipherStreamFromPrefix (
const std : : string & /*fname*/ , const EnvOptions & /*options*/ ,
uint64_t initialCounter , const Slice & iv , const Slice & /*prefix*/ ,
Yingchun Lai
2 years ago
committed by