@ -267,10 +267,11 @@ void ErrorHandler::CancelErrorRecovery() {
// This can also get called as part of a recovery operation. In that case, we
// This can also get called as part of a recovery operation. In that case, we
// also track the error separately in recovery_error_ so we can tell in the
// also track the error separately in recovery_error_ so we can tell in the
// end whether recovery succeeded or not
// end whether recovery succeeded or not
Status ErrorHandler : : SetBGError ( const Status & bg_err , BackgroundErrorReason reason ) {
const Status & ErrorHandler : : SetBGError ( const Status & bg_err ,
BackgroundErrorReason reason ) {
db_mutex_ - > AssertHeld ( ) ;
db_mutex_ - > AssertHeld ( ) ;
if ( bg_err . ok ( ) ) {
if ( bg_err . ok ( ) ) {
return Status : : OK ( ) ;
return bg_err ;
}
}
bool paranoid = db_options_ . paranoid_checks ;
bool paranoid = db_options_ . paranoid_checks ;
@ -363,11 +364,11 @@ Status ErrorHandler::SetBGError(const Status& bg_err, BackgroundErrorReason reas
// c) all other errors are mapped to hard error.
// c) all other errors are mapped to hard error.
// 3) for other cases, SetBGError(const Status& bg_err, BackgroundErrorReason
// 3) for other cases, SetBGError(const Status& bg_err, BackgroundErrorReason
// reason) will be called to handle other error cases.
// reason) will be called to handle other error cases.
Status ErrorHandler : : SetBGError ( const IOStatus & bg_io_err ,
const Status & ErrorHandler : : SetBGError ( const IOStatus & bg_io_err ,
BackgroundErrorReason reason ) {
BackgroundErrorReason reason ) {
db_mutex_ - > AssertHeld ( ) ;
db_mutex_ - > AssertHeld ( ) ;
if ( bg_io_err . ok ( ) ) {
if ( bg_io_err . ok ( ) ) {
return Status : : OK ( ) ;
return bg_io_err ;
}
}
ROCKS_LOG_WARN ( db_options_ . info_log , " Background IO error %s " ,
ROCKS_LOG_WARN ( db_options_ . info_log , " Background IO error %s " ,
bg_io_err . ToString ( ) . c_str ( ) ) ;
bg_io_err . ToString ( ) . c_str ( ) ) ;
@ -382,7 +383,6 @@ Status ErrorHandler::SetBGError(const IOStatus& bg_io_err,
}
}
Status new_bg_io_err = bg_io_err ;
Status new_bg_io_err = bg_io_err ;
Status s ;
DBRecoverContext context ;
DBRecoverContext context ;
if ( bg_io_err . GetDataLoss ( ) ) {
if ( bg_io_err . GetDataLoss ( ) ) {
// First, data loss is treated as unrecoverable error. So it can directly
// First, data loss is treated as unrecoverable error. So it can directly
@ -393,8 +393,8 @@ Status ErrorHandler::SetBGError(const IOStatus& bg_io_err,
if ( recovery_in_prog_ & & recovery_error_ . ok ( ) ) {
if ( recovery_in_prog_ & & recovery_error_ . ok ( ) ) {
recovery_error_ = bg_err ;
recovery_error_ = bg_err ;
}
}
EventHelpers : : NotifyOnBackgroundError ( db_options_ . listeners , reason , & s ,
EventHelpers : : NotifyOnBackgroundError ( db_options_ . listeners , reason ,
db_mutex_ , & auto_recovery ) ;
& bg_err , db_mutex_ , & auto_recovery ) ;
recover_context_ = context ;
recover_context_ = context ;
return bg_error_ ;
return bg_error_ ;
} else if ( bg_io_err . GetRetryable ( ) ) {
} else if ( bg_io_err . GetRetryable ( ) ) {
@ -405,8 +405,9 @@ Status ErrorHandler::SetBGError(const IOStatus& bg_io_err,
// soft error. In other cases, treat the retryable IO error as hard
// soft error. In other cases, treat the retryable IO error as hard
// error.
// error.
bool auto_recovery = false ;
bool auto_recovery = false ;
EventHelpers : : NotifyOnBackgroundError ( db_options_ . listeners , reason , & s ,
EventHelpers : : NotifyOnBackgroundError ( db_options_ . listeners , reason ,
db_mutex_ , & auto_recovery ) ;
& new_bg_io_err , db_mutex_ ,
& auto_recovery ) ;
if ( BackgroundErrorReason : : kCompaction = = reason ) {
if ( BackgroundErrorReason : : kCompaction = = reason ) {
Status bg_err ( new_bg_io_err , Status : : Severity : : kSoftError ) ;
Status bg_err ( new_bg_io_err , Status : : Severity : : kSoftError ) ;
if ( bg_err . severity ( ) > bg_error_ . severity ( ) ) {
if ( bg_err . severity ( ) > bg_error_ . severity ( ) ) {
@ -446,12 +447,11 @@ Status ErrorHandler::SetBGError(const IOStatus& bg_io_err,
return StartRecoverFromRetryableBGIOError ( bg_io_err ) ;
return StartRecoverFromRetryableBGIOError ( bg_io_err ) ;
}
}
} else {
} else {
s = SetBGError ( new_bg_io_err , reason ) ;
return SetBGError ( new_bg_io_err , reason ) ;
}
}
return s ;
}
}
Status ErrorHandler : : OverrideNoSpaceError ( Status bg_error ,
Status ErrorHandler : : OverrideNoSpaceError ( const Status & bg_error ,
bool * auto_recovery ) {
bool * auto_recovery ) {
# ifndef ROCKSDB_LITE
# ifndef ROCKSDB_LITE
if ( bg_error . severity ( ) > = Status : : Severity : : kFatalError ) {
if ( bg_error . severity ( ) > = Status : : Severity : : kFatalError ) {
@ -507,7 +507,11 @@ Status ErrorHandler::ClearBGError() {
// Signal that recovery succeeded
// Signal that recovery succeeded
if ( recovery_error_ . ok ( ) ) {
if ( recovery_error_ . ok ( ) ) {
Status old_bg_error = bg_error_ ;
Status old_bg_error = bg_error_ ;
// Clear and check the recovery IO and BG error
bg_error_ = Status : : OK ( ) ;
bg_error_ = Status : : OK ( ) ;
recovery_io_error_ = IOStatus : : OK ( ) ;
bg_error_ . PermitUncheckedError ( ) ;
recovery_io_error_ . PermitUncheckedError ( ) ;
recovery_in_prog_ = false ;
recovery_in_prog_ = false ;
soft_error_no_bg_work_ = false ;
soft_error_no_bg_work_ = false ;
EventHelpers : : NotifyOnErrorRecoveryCompleted ( db_options_ . listeners ,
EventHelpers : : NotifyOnErrorRecoveryCompleted ( db_options_ . listeners ,
@ -557,6 +561,7 @@ Status ErrorHandler::RecoverFromBGError(bool is_manual) {
// during the recovery process. While recovering, the only operations that
// during the recovery process. While recovering, the only operations that
// can generate background errors should be the flush operations
// can generate background errors should be the flush operations
recovery_error_ = Status : : OK ( ) ;
recovery_error_ = Status : : OK ( ) ;
recovery_error_ . PermitUncheckedError ( ) ;
Status s = db_ - > ResumeImpl ( recover_context_ ) ;
Status s = db_ - > ResumeImpl ( recover_context_ ) ;
if ( s . ok ( ) ) {
if ( s . ok ( ) ) {
soft_error_no_bg_work_ = false ;
soft_error_no_bg_work_ = false ;
@ -578,13 +583,15 @@ Status ErrorHandler::RecoverFromBGError(bool is_manual) {
# endif
# endif
}
}
Status ErrorHandler : : StartRecoverFromRetryableBGIOError ( IOStatus io_error ) {
const Status & ErrorHandler : : StartRecoverFromRetryableBGIOError (
const IOStatus & io_error ) {
# ifndef ROCKSDB_LITE
# ifndef ROCKSDB_LITE
db_mutex_ - > AssertHeld ( ) ;
db_mutex_ - > AssertHeld ( ) ;
if ( bg_error_ . ok ( ) | | io_error . ok ( ) ) {
if ( bg_error_ . ok ( ) ) {
return Status : : OK ( ) ;
return bg_error_ ;
}
} else if ( io_error . ok ( ) ) {
if ( db_options_ . max_bgerror_resume_count < = 0 | | recovery_in_prog_ ) {
return io_error ;
} else if ( db_options_ . max_bgerror_resume_count < = 0 | | recovery_in_prog_ ) {
// Auto resume BG error is not enabled, directly return bg_error_.
// Auto resume BG error is not enabled, directly return bg_error_.
return bg_error_ ;
return bg_error_ ;
}
}
@ -603,7 +610,7 @@ Status ErrorHandler::StartRecoverFromRetryableBGIOError(IOStatus io_error) {
new port : : Thread ( & ErrorHandler : : RecoverFromRetryableBGIOError , this ) ) ;
new port : : Thread ( & ErrorHandler : : RecoverFromRetryableBGIOError , this ) ) ;
if ( recovery_io_error_ . ok ( ) & & recovery_error_ . ok ( ) ) {
if ( recovery_io_error_ . ok ( ) & & recovery_error_ . ok ( ) ) {
return Status : : OK ( ) ;
return recovery_error_ ;
} else {
} else {
TEST_SYNC_POINT ( " StartRecoverRetryableBGIOError:RecoverFail " ) ;
TEST_SYNC_POINT ( " StartRecoverRetryableBGIOError:RecoverFail " ) ;
return bg_error_ ;
return bg_error_ ;
@ -668,6 +675,7 @@ void ErrorHandler::RecoverFromRetryableBGIOError() {
TEST_SYNC_POINT ( " RecoverFromRetryableBGIOError:RecoverSuccess " ) ;
TEST_SYNC_POINT ( " RecoverFromRetryableBGIOError:RecoverSuccess " ) ;
Status old_bg_error = bg_error_ ;
Status old_bg_error = bg_error_ ;
bg_error_ = Status : : OK ( ) ;
bg_error_ = Status : : OK ( ) ;
bg_error_ . PermitUncheckedError ( ) ;
EventHelpers : : NotifyOnErrorRecoveryCompleted ( db_options_ . listeners ,
EventHelpers : : NotifyOnErrorRecoveryCompleted ( db_options_ . listeners ,
old_bg_error , db_mutex_ ) ;
old_bg_error , db_mutex_ ) ;
recovery_in_prog_ = false ;
recovery_in_prog_ = false ;