@ -404,82 +404,91 @@ std::shared_ptr<BlobFile> BlobDBImpl::FindBlobFileLocked(
return ( b1 | | b2 ) ? nullptr : ( * finditr ) ;
return ( b1 | | b2 ) ? nullptr : ( * finditr ) ;
}
}
std : : shared_ptr < Writer > BlobDBImpl : : CheckOrCreateWriterLocked (
Status BlobDBImpl : : CheckOrCreateWriterLocked (
const std : : shared_ptr < BlobFile > & bfile ) {
const std : : shared_ptr < BlobFile > & blob_file ,
std : : shared_ptr < Writer > writer = bfile - > GetWriter ( ) ;
std : : shared_ptr < Writer > * writer ) {
if ( writer ) return writer ;
assert ( writer ! = nullptr ) ;
* writer = blob_file - > GetWriter ( ) ;
Status s = CreateWriterLocked ( bfile ) ;
if ( * writer ! = nullptr ) {
if ( ! s . ok ( ) ) return nullptr ;
return Status : : OK ( ) ;
}
writer = bfile - > GetWriter ( ) ;
Status s = CreateWriterLocked ( blob_file ) ;
return writer ;
if ( s . ok ( ) ) {
* writer = blob_file - > GetWriter ( ) ;
}
return s ;
}
}
std : : shared_ptr < BlobFile > BlobDBImpl : : SelectBlobFile ( ) {
Status BlobDBImpl : : SelectBlobFile ( std : : shared_ptr < BlobFile > * blob_file ) {
assert ( blob_file ! = nullptr ) ;
{
{
ReadLock rl ( & mutex_ ) ;
ReadLock rl ( & mutex_ ) ;
if ( open_non_ttl_file_ ! = nullptr ) {
if ( open_non_ttl_file_ ! = nullptr ) {
return open_non_ttl_file_ ;
* blob_file = open_non_ttl_file_ ;
return Status : : OK ( ) ;
}
}
}
}
// CHECK again
// CHECK again
WriteLock wl ( & mutex_ ) ;
WriteLock wl ( & mutex_ ) ;
if ( open_non_ttl_file_ ! = nullptr ) {
if ( open_non_ttl_file_ ! = nullptr ) {
return open_non_ttl_file_ ;
* blob_file = open_non_ttl_file_ ;
return Status : : OK ( ) ;
}
}
std : : shared_ptr < BlobFile > b file = NewBlobFile ( " SelectBlobFile " ) ;
* blob_ file = NewBlobFile ( " SelectBlobFile " ) ;
assert ( bfile ) ;
assert ( * blob_ file ! = nullptr ) ;
// file not visible, hence no lock
// file not visible, hence no lock
std : : shared_ptr < Writer > writer = CheckOrCreateWriterLocked ( bfile ) ;
std : : shared_ptr < Writer > writer ;
if ( ! writer ) {
Status s = CheckOrCreateWriterLocked ( * blob_file , & writer ) ;
if ( ! s . ok ( ) ) {
ROCKS_LOG_ERROR ( db_options_ . info_log ,
ROCKS_LOG_ERROR ( db_options_ . info_log ,
" Failed to get writer from blob file: %s " ,
" Failed to get writer from blob file: %s, error: %s " ,
bfile - > PathName ( ) . c_str ( ) ) ;
( * blob_ file ) - > PathName ( ) . c_str ( ) , s . ToString ( ) . c_str ( ) ) ;
return nullptr ;
return s ;
}
}
bfile - > file_size_ = BlobLogHeader : : kSize ;
( * blob_ file ) - > file_size_ = BlobLogHeader : : kSize ;
bfile - > header_ . compression = bdb_options_ . compression ;
( * blob_ file ) - > header_ . compression = bdb_options_ . compression ;
bfile - > header_ . has_ttl = false ;
( * blob_ file ) - > header_ . has_ttl = false ;
bfile - > header_ . column_family_id =
( * blob_ file ) - > header_ . column_family_id =
reinterpret_cast < ColumnFamilyHandleImpl * > ( DefaultColumnFamily ( ) ) - > GetID ( ) ;
reinterpret_cast < ColumnFamilyHandleImpl * > ( DefaultColumnFamily ( ) ) - > GetID ( ) ;
bfile - > header_valid_ = true ;
( * blob_ file ) - > header_valid_ = true ;
bfile - > SetColumnFamilyId ( bfile - > header_ . column_family_id ) ;
( * blob_ file ) - > SetColumnFamilyId ( ( * blob_ file ) - > header_ . column_family_id ) ;
bfile - > SetHasTTL ( false ) ;
( * blob_ file ) - > SetHasTTL ( false ) ;
bfile - > SetCompression ( bdb_options_ . compression ) ;
( * blob_ file ) - > SetCompression ( bdb_options_ . compression ) ;
Status s = writer - > WriteHeader ( bfile - > header_ ) ;
s = writer - > WriteHeader ( ( * blob_ file ) - > header_ ) ;
if ( ! s . ok ( ) ) {
if ( ! s . ok ( ) ) {
ROCKS_LOG_ERROR ( db_options_ . info_log ,
ROCKS_LOG_ERROR ( db_options_ . info_log ,
" Failed to write header to new blob file: %s "
" Failed to write header to new blob file: %s "
" status: '%s' " ,
" status: '%s' " ,
bfile - > PathName ( ) . c_str ( ) , s . ToString ( ) . c_str ( ) ) ;
( * blob_ file ) - > PathName ( ) . c_str ( ) , s . ToString ( ) . c_str ( ) ) ;
return nullptr ;
return s ;
}
}
blob_files_ . insert ( std : : make_pair ( bfile - > BlobFileNumber ( ) , bfile ) ) ;
blob_files_ . insert (
open_non_ttl_file_ = bfile ;
std : : make_pair ( ( * blob_file ) - > BlobFileNumber ( ) , * blob_file ) ) ;
open_non_ttl_file_ = * blob_file ;
total_blob_size_ + = BlobLogHeader : : kSize ;
total_blob_size_ + = BlobLogHeader : : kSize ;
return bfile ;
return s ;
}
}
std : : shared_ptr < BlobFile > BlobDBImpl : : SelectBlobFileTTL ( uint64_t expiration ) {
Status BlobDBImpl : : SelectBlobFileTTL ( uint64_t expiration ,
std : : shared_ptr < BlobFile > * blob_file ) {
assert ( blob_file ! = nullptr ) ;
assert ( expiration ! = kNoExpiration ) ;
assert ( expiration ! = kNoExpiration ) ;
uint64_t epoch_read = 0 ;
uint64_t epoch_read = 0 ;
std : : shared_ptr < BlobFile > bfile ;
{
{
ReadLock rl ( & mutex_ ) ;
ReadLock rl ( & mutex_ ) ;
bfile = FindBlobFileLocked ( expiration ) ;
* blob_ file = FindBlobFileLocked ( expiration ) ;
epoch_read = epoch_of_ . load ( ) ;
epoch_read = epoch_of_ . load ( ) ;
}
}
if ( bfile ) {
if ( * blob_ file ! = nullptr ) {
assert ( ! bfile - > Immutable ( ) ) ;
assert ( ! ( * blob_ file ) - > Immutable ( ) ) ;
return bfile ;
return Status : : OK ( ) ;
}
}
uint64_t exp_low =
uint64_t exp_low =
@ -487,61 +496,66 @@ std::shared_ptr<BlobFile> BlobDBImpl::SelectBlobFileTTL(uint64_t expiration) {
uint64_t exp_high = exp_low + bdb_options_ . ttl_range_secs ;
uint64_t exp_high = exp_low + bdb_options_ . ttl_range_secs ;
ExpirationRange expiration_range = std : : make_pair ( exp_low , exp_high ) ;
ExpirationRange expiration_range = std : : make_pair ( exp_low , exp_high ) ;
bfile = NewBlobFile ( " SelectBlobFileTTL " ) ;
* blob_ file = NewBlobFile ( " SelectBlobFileTTL " ) ;
assert ( bfile ) ;
assert ( * blob_ file ! = nullptr ) ;
ROCKS_LOG_INFO ( db_options_ . info_log , " New blob file TTL range: %s %d %d " ,
ROCKS_LOG_INFO ( db_options_ . info_log , " New blob file TTL range: %s %d %d " ,
bfile - > PathName ( ) . c_str ( ) , exp_low , exp_high ) ;
( * blob_ file ) - > PathName ( ) . c_str ( ) , exp_low , exp_high ) ;
LogFlush ( db_options_ . info_log ) ;
LogFlush ( db_options_ . info_log ) ;
// we don't need to take lock as no other thread is seeing bfile yet
// we don't need to take lock as no other thread is seeing bfile yet
std : : shared_ptr < Writer > writer = CheckOrCreateWriterLocked ( bfile ) ;
std : : shared_ptr < Writer > writer ;
if ( ! writer ) {
Status s = CheckOrCreateWriterLocked ( * blob_file , & writer ) ;
ROCKS_LOG_ERROR ( db_options_ . info_log ,
if ( ! s . ok ( ) ) {
" Failed to get writer from blob file with TTL: %s " ,
ROCKS_LOG_ERROR (
bfile - > PathName ( ) . c_str ( ) ) ;
db_options_ . info_log ,
return nullptr ;
" Failed to get writer from blob file with TTL: %s, error: %s " ,
( * blob_file ) - > PathName ( ) . c_str ( ) , s . ToString ( ) . c_str ( ) ) ;
return s ;
}
}
bfile - > header_ . expiration_range = expiration_range ;
( * blob_ file ) - > header_ . expiration_range = expiration_range ;
bfile - > header_ . compression = bdb_options_ . compression ;
( * blob_ file ) - > header_ . compression = bdb_options_ . compression ;
bfile - > header_ . has_ttl = true ;
( * blob_ file ) - > header_ . has_ttl = true ;
bfile - > header_ . column_family_id =
( * blob_ file ) - > header_ . column_family_id =
reinterpret_cast < ColumnFamilyHandleImpl * > ( DefaultColumnFamily ( ) ) - > GetID ( ) ;
reinterpret_cast < ColumnFamilyHandleImpl * > ( DefaultColumnFamily ( ) ) - > GetID ( ) ;
;
( * blob_file ) - > header_valid_ = true ;
bfile - > header_valid_ = true ;
( * blob_file ) - > SetColumnFamilyId ( ( * blob_file ) - > header_ . column_family_id ) ;
bfile - > SetColumnFamilyId ( bfile - > header_ . column_family_id ) ;
( * blob_file ) - > SetHasTTL ( true ) ;
bfile - > SetHasTTL ( true ) ;
( * blob_file ) - > SetCompression ( bdb_options_ . compression ) ;
bfile - > SetCompression ( bdb_options_ . compression ) ;
( * blob_file ) - > file_size_ = BlobLogHeader : : kSize ;
bfile - > file_size_ = BlobLogHeader : : kSize ;
// set the first value of the range, since that is
// set the first value of the range, since that is
// concrete at this time. also necessary to add to open_ttl_files_
// concrete at this time. also necessary to add to open_ttl_files_
bfile - > expiration_range_ = expiration_range ;
( * blob_ file ) - > expiration_range_ = expiration_range ;
WriteLock wl ( & mutex_ ) ;
WriteLock wl ( & mutex_ ) ;
// in case the epoch has shifted in the interim, then check
// in case the epoch has shifted in the interim, then check
// check condition again - should be rare.
// check condition again - should be rare.
if ( epoch_of_ . load ( ) ! = epoch_read ) {
if ( epoch_of_ . load ( ) ! = epoch_read ) {
auto bfile2 = FindBlobFileLocked ( expiration ) ;
std : : shared_ptr < BlobFile > blob_file2 = FindBlobFileLocked ( expiration ) ;
if ( bfile2 ) return bfile2 ;
if ( blob_file2 ! = nullptr ) {
* blob_file = std : : move ( blob_file2 ) ;
return Status : : OK ( ) ;
}
}
}
Status s = writer - > WriteHeader ( bfile - > header_ ) ;
s = writer - > WriteHeader ( ( * blob_ file ) - > header_ ) ;
if ( ! s . ok ( ) ) {
if ( ! s . ok ( ) ) {
ROCKS_LOG_ERROR ( db_options_ . info_log ,
ROCKS_LOG_ERROR ( db_options_ . info_log ,
" Failed to write header to new blob file: %s "
" Failed to write header to new blob file: %s "
" status: '%s' " ,
" status: '%s' " ,
bfile - > PathName ( ) . c_str ( ) , s . ToString ( ) . c_str ( ) ) ;
( * blob_ file ) - > PathName ( ) . c_str ( ) , s . ToString ( ) . c_str ( ) ) ;
return nullptr ;
return s ;
}
}
blob_files_ . insert ( std : : make_pair ( bfile - > BlobFileNumber ( ) , bfile ) ) ;
blob_files_ . insert (
open_ttl_files_ . insert ( bfile ) ;
std : : make_pair ( ( * blob_file ) - > BlobFileNumber ( ) , * blob_file ) ) ;
open_ttl_files_ . insert ( * blob_file ) ;
total_blob_size_ + = BlobLogHeader : : kSize ;
total_blob_size_ + = BlobLogHeader : : kSize ;
epoch_of_ + + ;
epoch_of_ + + ;
return bfile ;
return s ;
}
}
class BlobDBImpl : : BlobInserter : public WriteBatch : : Handler {
class BlobDBImpl : : BlobInserter : public WriteBatch : : Handler {
@ -695,36 +709,41 @@ Status BlobDBImpl::PutBlobValue(const WriteOptions& /*options*/,
return s ;
return s ;
}
}
std : : shared_ptr < BlobFile > bfile = ( expiration ! = kNoExpiration )
std : : shared_ptr < BlobFile > blob_file ;
? SelectBlobFileTTL ( expiration )
if ( expiration ! = kNoExpiration ) {
: SelectBlobFile ( ) ;
s = SelectBlobFileTTL ( expiration , & blob_file ) ;
assert ( bfile ! = nullptr ) ;
assert ( bfile - > compression ( ) = = bdb_options_ . compression ) ;
s = AppendBlob ( bfile , headerbuf , key , value_compressed , expiration ,
& index_entry ) ;
if ( expiration = = kNoExpiration ) {
RecordTick ( statistics_ , BLOB_DB_WRITE_BLOB ) ;
} else {
} else {
RecordTick ( statistics_ , BLOB_DB_WRITE_BLOB_TTL ) ;
s = SelectBlobFile ( & blob_file ) ;
}
if ( s . ok ( ) ) {
assert ( blob_file ! = nullptr ) ;
assert ( blob_file - > compression ( ) = = bdb_options_ . compression ) ;
s = AppendBlob ( blob_file , headerbuf , key , value_compressed , expiration ,
& index_entry ) ;
}
}
if ( s . ok ( ) ) {
if ( s . ok ( ) ) {
if ( expiration ! = kNoExpiration ) {
if ( expiration ! = kNoExpiration ) {
bfile - > ExtendExpirationRange ( expiration ) ;
blob_file - > ExtendExpirationRange ( expiration ) ;
}
s = CloseBlobFileIfNeeded ( blob_file ) ;
}
}
s = CloseBlobFileIfNeeded ( bfile ) ;
if ( s . ok ( ) ) {
if ( s . ok ( ) ) {
s = WriteBatchInternal : : PutBlobIndex ( batch , column_family_id , key ,
s = WriteBatchInternal : : PutBlobIndex ( batch , column_family_id , key ,
index_entry ) ;
index_entry ) ;
}
}
if ( s . ok ( ) ) {
if ( expiration = = kNoExpiration ) {
RecordTick ( statistics_ , BLOB_DB_WRITE_BLOB ) ;
} else {
RecordTick ( statistics_ , BLOB_DB_WRITE_BLOB_TTL ) ;
}
} else {
} else {
ROCKS_LOG_ERROR ( db_options_ . info_log ,
ROCKS_LOG_ERROR ( db_options_ . info_log ,
" Failed to append blob to FILE: %s: KEY: %s VALSZ: %d "
" Failed to append blob to FILE: %s: KEY: %s VALSZ: %d "
" status: '%s' blob_file: '%s' " ,
" status: '%s' blob_file: '%s' " ,
bfile - > PathName ( ) . c_str ( ) , key . ToString ( ) . c_str ( ) ,
blob_ file - > PathName ( ) . c_str ( ) , key . ToString ( ) . c_str ( ) ,
value . size ( ) , s . ToString ( ) . c_str ( ) ,
value . size ( ) , s . ToString ( ) . c_str ( ) ,
bfile - > DumpState ( ) . c_str ( ) ) ;
blob_ file - > DumpState ( ) . c_str ( ) ) ;
}
}
}
}
@ -867,9 +886,10 @@ Status BlobDBImpl::AppendBlob(const std::shared_ptr<BlobFile>& bfile,
uint64_t key_offset = 0 ;
uint64_t key_offset = 0 ;
{
{
WriteLock lockbfile_w ( & bfile - > mutex_ ) ;
WriteLock lockbfile_w ( & bfile - > mutex_ ) ;
std : : shared_ptr < Writer > writer = CheckOrCreateWriterLocked ( bfile ) ;
std : : shared_ptr < Writer > writer ;
if ( ! writer ) {
s = CheckOrCreateWriterLocked ( bfile , & writer ) ;
return Status : : IOError ( " Failed to create blob writer " ) ;
if ( ! s . ok ( ) ) {
return s ;
}
}
// write the blob to the blob log.
// write the blob to the blob log.
@ -1574,7 +1594,13 @@ Status BlobDBImpl::GCFileAndUpdateLSM(const std::shared_ptr<BlobFile>& bfptr,
reason + = bfptr - > PathName ( ) ;
reason + = bfptr - > PathName ( ) ;
newfile = NewBlobFile ( reason ) ;
newfile = NewBlobFile ( reason ) ;
new_writer = CheckOrCreateWriterLocked ( newfile ) ;
s = CheckOrCreateWriterLocked ( newfile , & new_writer ) ;
if ( ! s . ok ( ) ) {
ROCKS_LOG_ERROR ( db_options_ . info_log ,
" Failed to open file %s for writer, error: %s " ,
newfile - > PathName ( ) . c_str ( ) , s . ToString ( ) . c_str ( ) ) ;
break ;
}
// Can't use header beyond this point
// Can't use header beyond this point
newfile - > header_ = std : : move ( header ) ;
newfile - > header_ = std : : move ( header ) ;
newfile - > header_valid_ = true ;
newfile - > header_valid_ = true ;