@ -178,20 +178,13 @@ class VectorIterator : public InternalIterator {
std : : vector < std : : string > values_ ;
size_t current_ ;
} ;
extern WritableFileWriter * GetWritableFileWriter ( WritableFile * wf ,
const std : : string & fname ) ;
extern RandomAccessFileReader * GetRandomAccessFileReader ( RandomAccessFile * raf ) ;
extern SequentialFileReader * GetSequentialFileReader ( SequentialFile * se ,
const std : : string & fname ) ;
class StringSink : public WritableFile {
class StringSink : public FSWritableFile {
public :
std : : string contents_ ;
explicit StringSink ( Slice * reader_contents = nullptr ) :
WritableFile ( ) ,
explicit StringSink ( Slice * reader_contents = nullptr )
: FSWritableFile ( ) ,
contents_ ( " " ) ,
reader_contents_ ( reader_contents ) ,
last_flush_ ( 0 ) {
@ -202,12 +195,15 @@ class StringSink: public WritableFile {
const std : : string & contents ( ) const { return contents_ ; }
virtual Status Truncate ( uint64_t size ) override {
IOStatus Truncate ( uint64_t size , const IOOptions & /*opts*/ ,
IODebugContext * /*dbg*/ ) override {
contents_ . resize ( static_cast < size_t > ( size ) ) ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
virtual Status Close ( ) override { return Status : : OK ( ) ; }
virtual Status Flush ( ) override {
IOStatus Close ( const IOOptions & /*opts*/ , IODebugContext * /*dbg*/ ) override {
return IOStatus : : OK ( ) ;
}
IOStatus Flush ( const IOOptions & /*opts*/ , IODebugContext * /*dbg*/ ) override {
if ( reader_contents_ ! = nullptr ) {
assert ( reader_contents_ - > size ( ) < = last_flush_ ) ;
size_t offset = last_flush_ - reader_contents_ - > size ( ) ;
@ -217,12 +213,17 @@ class StringSink: public WritableFile {
last_flush_ = contents_ . size ( ) ;
}
return Status : : OK ( ) ;
return IOStatus : : OK ( ) ;
}
IOStatus Sync ( const IOOptions & /*opts*/ , IODebugContext * /*dbg*/ ) override {
return IOStatus : : OK ( ) ;
}
virtual Status Sync ( ) override { return Status : : OK ( ) ; }
virtual Status Append ( const Slice & slice ) override {
using FSWritableFile : : Append ;
IOStatus Append ( const Slice & slice , const IOOptions & /*opts*/ ,
IODebugContext * /*dbg*/ ) override {
contents_ . append ( slice . data ( ) , slice . size ( ) ) ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
void Drop ( size_t bytes ) {
if ( reader_contents_ ! = nullptr ) {
@ -239,36 +240,44 @@ class StringSink: public WritableFile {
} ;
// A wrapper around a StringSink to give it a RandomRWFile interface
class RandomRWStringSink : public RandomRWFile {
class RandomRWStringSink : public FS RandomRWFile{
public :
explicit RandomRWStringSink ( StringSink * ss ) : ss_ ( ss ) { }
Status Write ( uint64_t offset , const Slice & data ) override {
IOStatus Write ( uint64_t offset , const Slice & data , const IOOptions & /*opts*/ ,
IODebugContext * /*dbg*/ ) override {
if ( offset + data . size ( ) > ss_ - > contents_ . size ( ) ) {
ss_ - > contents_ . resize ( static_cast < size_t > ( offset ) + data . size ( ) , ' \0 ' ) ;
}
char * pos = const_cast < char * > ( ss_ - > contents_ . data ( ) + offset ) ;
memcpy ( pos , data . data ( ) , data . size ( ) ) ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
Status Read ( uint64_t offset , size_t n , Slice * result ,
char * /*scratch*/ ) const override {
IOStatus Read ( uint64_t offset , size_t n , const IOOptions & /*opts*/ ,
Slice * result , char * /*scratch*/ ,
IODebugContext * /*dbg*/ ) const override {
* result = Slice ( nullptr , 0 ) ;
if ( offset < ss_ - > contents_ . size ( ) ) {
size_t str_res_sz =
std : : min ( static_cast < size_t > ( ss_ - > contents_ . size ( ) - offset ) , n ) ;
* result = Slice ( ss_ - > contents_ . data ( ) + offset , str_res_sz ) ;
}
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
Status Flush ( ) override { return Status : : OK ( ) ; }
IOStatus Flush ( const IOOptions & /*opts*/ , IODebugContext * /*dbg*/ ) override {
return IOStatus : : OK ( ) ;
}
Status Sync ( ) override { return Status : : OK ( ) ; }
IOStatus Sync ( const IOOptions & /*opts*/ , IODebugContext * /*dbg*/ ) override {
return IOStatus : : OK ( ) ;
}
Status Close ( ) override { return Status : : OK ( ) ; }
IOStatus Close ( const IOOptions & /*opts*/ , IODebugContext * /*dbg*/ ) override {
return IOStatus : : OK ( ) ;
}
const std : : string & contents ( ) const { return ss_ - > contents ( ) ; }
@ -279,34 +288,42 @@ class RandomRWStringSink : public RandomRWFile {
// Like StringSink, this writes into a string. Unlink StringSink, it
// has some initial content and overwrites it, just like a recycled
// log file.
class OverwritingStringSink : public WritableFile {
class OverwritingStringSink : public FS WritableFile{
public :
explicit OverwritingStringSink ( Slice * reader_contents )
: WritableFile ( ) ,
: FS WritableFile( ) ,
contents_ ( " " ) ,
reader_contents_ ( reader_contents ) ,
last_flush_ ( 0 ) { }
const std : : string & contents ( ) const { return contents_ ; }
virtual Status Truncate ( uint64_t size ) override {
IOStatus Truncate ( uint64_t size , const IOOptions & /*opts*/ ,
IODebugContext * /*dbg*/ ) override {
contents_ . resize ( static_cast < size_t > ( size ) ) ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
virtual Status Close ( ) override { return Status : : OK ( ) ; }
virtual Status Flush ( ) override {
IOStatus Close ( const IOOptions & /*opts*/ , IODebugContext * /*dbg*/ ) override {
return IOStatus : : OK ( ) ;
}
IOStatus Flush ( const IOOptions & /*opts*/ , IODebugContext * /*dbg*/ ) override {
if ( last_flush_ < contents_ . size ( ) ) {
assert ( reader_contents_ - > size ( ) > = contents_ . size ( ) ) ;
memcpy ( ( char * ) reader_contents_ - > data ( ) + last_flush_ ,
contents_ . data ( ) + last_flush_ , contents_ . size ( ) - last_flush_ ) ;
last_flush_ = contents_ . size ( ) ;
}
return Status : : OK ( ) ;
return IOStatus : : OK ( ) ;
}
IOStatus Sync ( const IOOptions & /*opts*/ , IODebugContext * /*dbg*/ ) override {
return IOStatus : : OK ( ) ;
}
virtual Status Sync ( ) override { return Status : : OK ( ) ; }
virtual Status Append ( const Slice & slice ) override {
using FSWritableFile : : Append ;
IOStatus Append ( const Slice & slice , const IOOptions & /*opts*/ ,
IODebugContext * /*dbg*/ ) override {
contents_ . append ( slice . data ( ) , slice . size ( ) ) ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
void Drop ( size_t bytes ) {
contents_ . resize ( contents_ . size ( ) - bytes ) ;
@ -319,7 +336,7 @@ class OverwritingStringSink : public WritableFile {
size_t last_flush_ ;
} ;
class StringSource : public RandomAccessFile {
class StringSource : public FS RandomAccessFile{
public :
explicit StringSource ( const Slice & contents , uint64_t uniq_id = 0 ,
bool mmap = false )
@ -332,11 +349,23 @@ class StringSource: public RandomAccessFile {
uint64_t Size ( ) const { return contents_ . size ( ) ; }
virtual Status Read ( uint64_t offset , size_t n , Slice * result ,
char * scratch ) const override {
IOStatus Prefetch ( uint64_t /*offset*/ , size_t /*n*/ ,
const IOOptions & /*options*/ ,
IODebugContext * /*dbg*/ ) override {
// If we are using mmap_, it is equivalent to performing a prefetch
if ( mmap_ ) {
return IOStatus : : OK ( ) ;
} else {
return IOStatus : : NotSupported ( " Prefetch not supported " ) ;
}
}
IOStatus Read ( uint64_t offset , size_t n , const IOOptions & /*opts*/ ,
Slice * result , char * scratch ,
IODebugContext * /*dbg*/ ) const override {
total_reads_ + + ;
if ( offset > contents_ . size ( ) ) {
return Status : : InvalidArgument ( " invalid Read offset " ) ;
return IO Status: : InvalidArgument ( " invalid Read offset " ) ;
}
if ( offset + n > contents_ . size ( ) ) {
n = contents_ . size ( ) - static_cast < size_t > ( offset ) ;
@ -347,10 +376,10 @@ class StringSource: public RandomAccessFile {
} else {
* result = Slice ( & contents_ [ static_cast < size_t > ( offset ) ] , n ) ;
}
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
virtual size_t GetUniqueId ( char * id , size_t max_size ) const override {
size_t GetUniqueId ( char * id , size_t max_size ) const override {
if ( max_size < 20 ) {
return 0 ;
}
@ -372,13 +401,6 @@ class StringSource: public RandomAccessFile {
mutable int total_reads_ ;
} ;
inline StringSink * GetStringSinkFromLegacyWriter (
const WritableFileWriter * writer ) {
LegacyWritableFileWrapper * file =
static_cast < LegacyWritableFileWrapper * > ( writer - > writable_file ( ) ) ;
return static_cast < StringSink * > ( file - > target ( ) ) ;
}
class NullLogger : public Logger {
public :
using Logger : : Logv ;
@ -525,12 +547,13 @@ inline std::string EncodeInt(uint64_t x) {
return result ;
}
class SeqStringSource : public SequentialFile {
class SeqStringSource : public FS SequentialFile{
public :
SeqStringSource ( const std : : string & data , std : : atomic < int > * read_count )
: data_ ( data ) , offset_ ( 0 ) , read_count_ ( read_count ) { }
~ SeqStringSource ( ) override { }
Status Read ( size_t n , Slice * result , char * scratch ) override {
IOStatus Read ( size_t n , const IOOptions & /*opts*/ , Slice * result ,
char * scratch , IODebugContext * /*dbg*/ ) override {
std : : string output ;
if ( offset_ < data_ . size ( ) ) {
n = std : : min ( data_ . size ( ) - offset_ , n ) ;
@ -538,20 +561,21 @@ inline std::string EncodeInt(uint64_t x) {
offset_ + = n ;
* result = Slice ( scratch , n ) ;
} else {
return Status : : InvalidArgument (
" Attemp to read when it already reached eof. " ) ;
return IO Status: : InvalidArgument (
" Attempt to read when it already reached eof. " ) ;
}
( * read_count_ ) + + ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
Status Skip ( uint64_t n ) override {
IOStatus Skip ( uint64_t n ) override {
if ( offset_ > = data_ . size ( ) ) {
return Status : : InvalidArgument (
" Attemp to read when it already reached eof. " ) ;
return IO Status: : InvalidArgument (
" Attempt to read when it already reached eof. " ) ;
}
// TODO(yhchiang): Currently doesn't handle the overflow case.
offset_ + = static_cast < size_t > ( n ) ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
private :
@ -560,134 +584,176 @@ inline std::string EncodeInt(uint64_t x) {
std : : atomic < int > * read_count_ ;
} ;
class StringEnv : public Env Wrapper {
class StringFS : public FileSystem Wrapper {
public :
class StringSink : public WritableFile {
class StringSink : public FS WritableFile{
public :
explicit StringSink ( std : : string * contents )
: WritableFile ( ) , contents_ ( contents ) { }
virtual Status Truncate ( uint64_t size ) override {
: FSWritableFile ( ) , contents_ ( contents ) { }
IOStatus Truncate ( uint64_t size , const IOOptions & /*opts*/ ,
IODebugContext * /*dbg*/ ) override {
contents_ - > resize ( static_cast < size_t > ( size ) ) ;
return Status : : OK ( ) ;
return IOStatus : : OK ( ) ;
}
IOStatus Close ( const IOOptions & /*opts*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : OK ( ) ;
}
IOStatus Flush ( const IOOptions & /*opts*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : OK ( ) ;
}
virtual Status Close ( ) override { return Status : : OK ( ) ; }
virtual Status Flush ( ) override { return Status : : OK ( ) ; }
virtual Status Sync ( ) override { return Status : : OK ( ) ; }
virtual Status Append ( const Slice & slice ) override {
IOStatus Sync ( const IOOptions & /*opts*/ , IODebugContext * /*dbg*/ ) override {
return IOStatus : : OK ( ) ;
}
using FSWritableFile : : Append ;
IOStatus Append ( const Slice & slice , const IOOptions & /*opts*/ ,
IODebugContext * /*dbg*/ ) override {
contents_ - > append ( slice . data ( ) , slice . size ( ) ) ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
private :
std : : string * contents_ ;
} ;
explicit StringEnv ( Env * t ) : EnvWrapper ( t ) { }
~ StringEnv ( ) override { }
explicit StringFS ( const std : : shared_ptr < FileSystem > & t )
: FileSystemWrapper ( t ) { }
~ StringFS ( ) override { }
const std : : string & GetContent ( const std : : string & f ) { return files_ [ f ] ; }
const Status WriteToNewFile ( const std : : string & file_name ,
const IO StatusWriteToNewFile ( const std : : string & file_name ,
const std : : string & content ) {
std : : unique_ptr < WritableFile > r ;
auto s = NewWritableFile ( file_name , & r , EnvOptions ( ) ) ;
std : : unique_ptr < FSWritableFile > r ;
FileOptions file_opts ;
IOOptions io_opts ;
auto s = NewWritableFile ( file_name , file_opts , & r , nullptr ) ;
if ( s . ok ( ) ) {
s = r - > Append ( content ) ;
s = r - > Append ( content , io_opts , nullptr ) ;
}
if ( s . ok ( ) ) {
s = r - > Flush ( ) ;
s = r - > Flush ( io_opts , nullptr ) ;
}
if ( s . ok ( ) ) {
s = r - > Close ( ) ;
s = r - > Close ( io_opts , nullptr ) ;
}
assert ( ! s . ok ( ) | | files_ [ file_name ] = = content ) ;
return s ;
}
// The following text is boilerplate that forwards all methods to target()
Status NewSequentialFile ( const std : : string & f ,
std : : unique_ptr < SequentialFile > * r ,
const EnvOptions & /*options*/ ) override {
IOStatus NewSequentialFile ( const std : : string & f ,
const FileOptions & /*options*/ ,
std : : unique_ptr < FSSequentialFile > * r ,
IODebugContext * /*dbg*/ ) override {
auto iter = files_ . find ( f ) ;
if ( iter = = files_ . end ( ) ) {
return Status : : NotFound ( " The specified file does not exist " , f ) ;
return IO Status: : NotFound ( " The specified file does not exist " , f ) ;
}
r - > reset ( new SeqStringSource ( iter - > second , & num_seq_file_read_ ) ) ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
Status NewRandomAccessFile ( const std : : string & /*f*/ ,
std : : unique_ptr < RandomAccessFile > * /*r*/ ,
const EnvOptions & /*options*/ ) override {
return Status : : NotSupported ( ) ;
IOStatus NewRandomAccessFile ( const std : : string & /*f*/ ,
const FileOptions & /*options*/ ,
std : : unique_ptr < FSRandomAccessFile > * /*r*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : NotSupported ( ) ;
}
Status NewWritableFile ( const std : : string & f ,
std : : unique_ptr < WritableFile > * r ,
const EnvOptions & /*options*/ ) override {
IOStatus NewWritableFile ( const std : : string & f , const FileOptions & /*options*/ ,
std : : unique_ptr < FSWritableFile > * r ,
IODebugContext * /*dbg*/ ) override {
auto iter = files_ . find ( f ) ;
if ( iter ! = files_ . end ( ) ) {
return Status : : IOError ( " The specified file already exists " , f ) ;
return IO Status: : IOError ( " The specified file already exists " , f ) ;
}
r - > reset ( new StringSink ( & files_ [ f ] ) ) ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
virtual Status NewDirectory (
const std : : string & /*name*/ ,
std : : unique_ptr < Directory > * /*result*/ ) override {
return Status : : NotSupported ( ) ;
IOStatus NewDirectory ( const std : : string & /*name*/ ,
const IOOptions & /*options*/ ,
std : : unique_ptr < FSDirectory > * /*result*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : NotSupported ( ) ;
}
Status FileExists ( const std : : string & f ) override {
IOStatus FileExists ( const std : : string & f , const IOOptions & /*options*/ ,
IODebugContext * /*dbg*/ ) override {
if ( files_ . find ( f ) = = files_ . end ( ) ) {
return Status : : NotFound ( ) ;
return IO Status: : NotFound ( ) ;
}
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
Status GetChildren ( const std : : string & /*dir*/ ,
std : : vector < std : : string > * /*r*/ ) override {
return Status : : NotSupported ( ) ;
IOStatus GetChildren ( const std : : string & /*dir*/ , const IOOptions & /*options*/ ,
std : : vector < std : : string > * /*r*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : NotSupported ( ) ;
}
Status DeleteFile ( const std : : string & f ) override {
IOStatus DeleteFile ( const std : : string & f , const IOOptions & /*options*/ ,
IODebugContext * /*dbg*/ ) override {
files_ . erase ( f ) ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
Status CreateDir ( const std : : string & /*d*/ ) override {
return Status : : NotSupported ( ) ;
IOStatus CreateDir ( const std : : string & /*d*/ , const IOOptions & /*options*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : NotSupported ( ) ;
}
Status CreateDirIfMissing ( const std : : string & /*d*/ ) override {
return Status : : NotSupported ( ) ;
IOStatus CreateDirIfMissing ( const std : : string & /*d*/ ,
const IOOptions & /*options*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : NotSupported ( ) ;
}
Status DeleteDir ( const std : : string & /*d*/ ) override {
return Status : : NotSupported ( ) ;
IOStatus DeleteDir ( const std : : string & /*d*/ , const IOOptions & /*options*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : NotSupported ( ) ;
}
Status GetFileSize ( const std : : string & f , uint64_t * s ) override {
IOStatus GetFileSize ( const std : : string & f , const IOOptions & /*options*/ ,
uint64_t * s , IODebugContext * /*dbg*/ ) override {
auto iter = files_ . find ( f ) ;
if ( iter = = files_ . end ( ) ) {
return Status : : NotFound ( " The specified file does not exist: " , f ) ;
return IO Status: : NotFound ( " The specified file does not exist: " , f ) ;
}
* s = iter - > second . size ( ) ;
return Status : : OK ( ) ;
return IO Status: : OK ( ) ;
}
Status GetFileModificationTime ( const std : : string & /*fname*/ ,
uint64_t * /*file_mtime*/ ) override {
return Status : : NotSupported ( ) ;
IOStatus GetFileModificationTime ( const std : : string & /*fname*/ ,
const IOOptions & /*options*/ ,
uint64_t * /*file_mtime*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : NotSupported ( ) ;
}
Status RenameFile ( const std : : string & /*s*/ ,
const std : : string & /*t*/ ) override {
return Status : : NotSupported ( ) ;
IOStatus RenameFile ( const std : : string & /*s*/ , const std : : string & /*t*/ ,
const IOOptions & /*options*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : NotSupported ( ) ;
}
Status LinkFile ( const std : : string & /*s*/ ,
const std : : string & /*t*/ ) override {
return Status : : NotSupported ( ) ;
IOStatus LinkFile ( const std : : string & /*s*/ , const std : : string & /*t*/ ,
const IOOptions & /*options*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : NotSupported ( ) ;
}
Status LockFile ( const std : : string & /*f*/ , FileLock * * /*l*/ ) override {
return Status : : NotSupported ( ) ;
IOStatus LockFile ( const std : : string & /*f*/ , const IOOptions & /*options*/ ,
FileLock * * /*l*/ , IODebugContext * /*dbg*/ ) override {
return IOStatus : : NotSupported ( ) ;
}
Status UnlockFile ( FileLock * /*l*/ ) override {
return Status : : NotSupported ( ) ;
IOStatus UnlockFile ( FileLock * /*l*/ , const IOOptions & /*options*/ ,
IODebugContext * /*dbg*/ ) override {
return IOStatus : : NotSupported ( ) ;
}
std : : atomic < int > num_seq_file_read_ ;
mrambacher
4 years ago
committed by