@ -10,6 +10,7 @@
# include "rocksdb/filter_policy.h" # include "rocksdb/filter_policy.h"
# include <array> # include <array>
# include <cstring>
# include <deque> # include <deque>
# include <limits> # include <limits>
# include <memory> # include <memory>
@ -47,15 +48,22 @@ Slice FinishAlwaysFalse(std::unique_ptr<const char[]>* /*buf*/) {
return Slice ( nullptr , 0 ) ; return Slice ( nullptr , 0 ) ;
} }
Slice FinishAlwaysTrue ( std : : unique_ptr < const char [ ] > * /*buf*/ ) {
return Slice ( " \0 \0 \0 \0 \0 \0 " , 6 ) ;
}
// Base class for filter builders using the XXH3 preview hash,
// Base class for filter builders using the XXH3 preview hash,
// also known as Hash64 or GetSliceHash64.
// also known as Hash64 or GetSliceHash64.
class XXPH3FilterBitsBuilder : public BuiltinFilterBitsBuilder { class XXPH3FilterBitsBuilder : public BuiltinFilterBitsBuilder {
public : public :
explicit XXPH3FilterBitsBuilder ( explicit XXPH3FilterBitsBuilder (
std : : atomic < int64_t > * aggregate_rounding_balance , std : : atomic < int64_t > * aggregate_rounding_balance ,
std : : shared_ptr < CacheReservationManager > cache_res_mgr ) std : : shared_ptr < CacheReservationManager > cache_res_mgr ,
bool detect_filter_construct_corruption )
: aggregate_rounding_balance_ ( aggregate_rounding_balance ) , : aggregate_rounding_balance_ ( aggregate_rounding_balance ) ,
cache_res_mgr_ ( cache_res_mgr ) { } cache_res_mgr_ ( cache_res_mgr ) ,
detect_filter_construct_corruption_ (
detect_filter_construct_corruption ) { }
~ XXPH3FilterBitsBuilder ( ) override { } ~ XXPH3FilterBitsBuilder ( ) override { }
@ -65,27 +73,34 @@ class XXPH3FilterBitsBuilder : public BuiltinFilterBitsBuilder {
// though only adjacent repetition, which we want to immediately
// though only adjacent repetition, which we want to immediately
// recognize and collapse for estimating true filter space
// recognize and collapse for estimating true filter space
// requirements.
// requirements.
if ( hash_entries_ . empty ( ) | | hash ! = hash_entries_ . back ( ) ) { if ( hash_entries_info_ . entries . empty ( ) | |
hash_entries_ . push_back ( hash ) ; hash ! = hash_entries_info_ . entries . back ( ) ) {
if ( detect_filter_construct_corruption_ ) {
hash_entries_info_ . xor_checksum ^ = hash ;
}
hash_entries_info_ . entries . push_back ( hash ) ;
if ( cache_res_mgr_ & & if ( cache_res_mgr_ & &
// Traditional rounding to whole bucket size
// Traditional rounding to whole bucket size
( ( hash_entries_ . size ( ) % kUint64tHashEntryCacheResBucketSize ) = = ( ( hash_entries_info_ . entries . size ( ) %
kUint64tHashEntryCacheResBucketSize ) = =
kUint64tHashEntryCacheResBucketSize / 2 ) ) { kUint64tHashEntryCacheResBucketSize / 2 ) ) {
hash_entry_cache_res_bucket_handles_ . emplace_back ( nullptr ) ; hash_entries_info_ . cache_res_bucket_handles . emplace_back ( nullptr ) ;
Status s = Status s =
cache_res_mgr_ cache_res_mgr_
- > MakeCacheReservation < CacheEntryRole : : kFilterConstruction > ( - > MakeCacheReservation < CacheEntryRole : : kFilterConstruction > (
kUint64tHashEntryCacheResBucketSize * sizeof ( hash ) , kUint64tHashEntryCacheResBucketSize * sizeof ( hash ) ,
& hash_entry_cache_res_bucket_handles_ . back ( ) ) ; & hash_entries_info_ . cache_res_bucket_handles . back ( ) ) ;
s . PermitUncheckedError ( ) ; s . PermitUncheckedError ( ) ;
} }
} }
} }
virtual size_t EstimateEntriesAdded ( ) override { virtual size_t EstimateEntriesAdded ( ) override {
return hash_entries_ . size ( ) ; return hash_entries_info_ . entries . size ( ) ;
} }
virtual Status MaybePostVerify ( const Slice & filter_content ) override ;
protected : protected :
static constexpr uint32_t kMetadataLen = 5 ; static constexpr uint32_t kMetadataLen = 5 ;
@ -96,13 +111,12 @@ class XXPH3FilterBitsBuilder : public BuiltinFilterBitsBuilder {
// For delegating between XXPH3FilterBitsBuilders
// For delegating between XXPH3FilterBitsBuilders
void SwapEntriesWith ( XXPH3FilterBitsBuilder * other ) { void SwapEntriesWith ( XXPH3FilterBitsBuilder * other ) {
std : : swap ( hash_entries_ , other - > hash_entries_ ) ; assert ( other ! = nullptr ) ;
if ( cache_res_mgr_ ) { hash_entries_info_ . Swap ( & ( other - > hash_entries_info_ ) ) ;
std : : swap ( hash_entry_cache_res_bucket_handles_ ,
other - > hash_entry_cache_res_bucket_handles_ ) ;
}
} }
void ResetEntries ( ) { hash_entries_info_ . Reset ( ) ; }
virtual size_t RoundDownUsableSpace ( size_t available_size ) = 0 ; virtual size_t RoundDownUsableSpace ( size_t available_size ) = 0 ;
// To choose size using malloc_usable_size, we have to actually allocate.
// To choose size using malloc_usable_size, we have to actually allocate.
@ -203,9 +217,32 @@ class XXPH3FilterBitsBuilder : public BuiltinFilterBitsBuilder {
return rv ; return rv ;
} }
// A deque avoids unnecessary copying of already-saved values
// TODO: Ideally we want to verify the hash entry
// and has near-minimal peak memory use.
// as it is added to the filter and eliminate this function
std : : deque < uint64_t > hash_entries_ ; // for speeding up and leaving fewer spaces for undetected memory/CPU
// corruption. For Ribbon Filter, it's bit harder.
// Possible solution:
// pass a custom iterator that tracks the xor checksum as
// it iterates to ResetAndFindSeedToSolve
Status MaybeVerifyHashEntriesChecksum ( ) {
if ( ! detect_filter_construct_corruption_ ) {
return Status : : OK ( ) ;
}
uint64_t actual_hash_entries_xor_checksum = 0 ;
for ( uint64_t h : hash_entries_info_ . entries ) {
actual_hash_entries_xor_checksum ^ = h ;
}
if ( actual_hash_entries_xor_checksum = = hash_entries_info_ . xor_checksum ) {
return Status : : OK ( ) ;
} else {
// Since these hash entries are corrupted and they will not be used
// anymore, we can reset them and release memory.
ResetEntries ( ) ;
return Status : : Corruption ( " Filter's hash entries checksum mismatched " ) ;
}
}
// See BloomFilterPolicy::aggregate_rounding_balance_. If nullptr,
// See BloomFilterPolicy::aggregate_rounding_balance_. If nullptr,
// always "round up" like historic behavior.
// always "round up" like historic behavior.
@ -214,17 +251,47 @@ class XXPH3FilterBitsBuilder : public BuiltinFilterBitsBuilder {
// For reserving memory used in (new) Bloom and Ribbon Filter construction
// For reserving memory used in (new) Bloom and Ribbon Filter construction
std : : shared_ptr < CacheReservationManager > cache_res_mgr_ ; std : : shared_ptr < CacheReservationManager > cache_res_mgr_ ;
// For managing cache reservation for buckets of hash entry in (new) Bloom and
// Ribbon Filter construction
std : : deque < std : : unique_ptr <
CacheReservationHandle < CacheEntryRole : : kFilterConstruction > > >
hash_entry_cache_res_bucket_handles_ ;
// For managing cache reservation for final filter in (new) Bloom and Ribbon
// For managing cache reservation for final filter in (new) Bloom and Ribbon
// Filter construction
// Filter construction
std : : deque < std : : unique_ptr < std : : deque < std : : unique_ptr <
CacheReservationHandle < CacheEntryRole : : kFilterConstruction > > > CacheReservationHandle < CacheEntryRole : : kFilterConstruction > > >
final_filter_cache_res_handles_ ; final_filter_cache_res_handles_ ;
bool detect_filter_construct_corruption_ ;
struct HashEntriesInfo {
// A deque avoids unnecessary copying of already-saved values
// and has near-minimal peak memory use.
std : : deque < uint64_t > entries ;
// If cache_res_mgr_ != nullptr,
// it manages cache reservation for buckets of hash entries in (new) Bloom
// or Ribbon Filter construction.
// Otherwise, it is empty.
std : : deque < std : : unique_ptr <
CacheReservationHandle < CacheEntryRole : : kFilterConstruction > > >
cache_res_bucket_handles ;
// If detect_filter_construct_corruption_ == true,
// it records the xor checksum of hash entries.
// Otherwise, it is 0.
uint64_t xor_checksum = 0 ;
void Swap ( HashEntriesInfo * other ) {
assert ( other ! = nullptr ) ;
std : : swap ( entries , other - > entries ) ;
std : : swap ( cache_res_bucket_handles , other - > cache_res_bucket_handles ) ;
std : : swap ( xor_checksum , other - > xor_checksum ) ;
}
void Reset ( ) {
entries . clear ( ) ;
cache_res_bucket_handles . clear ( ) ;
xor_checksum = 0 ;
}
} ;
HashEntriesInfo hash_entries_info_ ;
} ; } ;
// #################### FastLocalBloom implementation ################## //
// #################### FastLocalBloom implementation ################## //
@ -237,8 +304,10 @@ class FastLocalBloomBitsBuilder : public XXPH3FilterBitsBuilder {
explicit FastLocalBloomBitsBuilder ( explicit FastLocalBloomBitsBuilder (
const int millibits_per_key , const int millibits_per_key ,
std : : atomic < int64_t > * aggregate_rounding_balance , std : : atomic < int64_t > * aggregate_rounding_balance ,
std : : shared_ptr < CacheReservationManager > cache_res_mgr ) std : : shared_ptr < CacheReservationManager > cache_res_mgr ,
: XXPH3FilterBitsBuilder ( aggregate_rounding_balance , cache_res_mgr ) , bool detect_filter_construct_corruption )
: XXPH3FilterBitsBuilder ( aggregate_rounding_balance , cache_res_mgr ,
detect_filter_construct_corruption ) ,
millibits_per_key_ ( millibits_per_key ) { millibits_per_key_ ( millibits_per_key ) {
assert ( millibits_per_key > = 1000 ) ; assert ( millibits_per_key > = 1000 ) ;
} }
@ -249,25 +318,29 @@ class FastLocalBloomBitsBuilder : public XXPH3FilterBitsBuilder {
~ FastLocalBloomBitsBuilder ( ) override { } ~ FastLocalBloomBitsBuilder ( ) override { }
using FilterBitsBuilder : : Finish ;
virtual Slice Finish ( std : : unique_ptr < const char [ ] > * buf ) override { virtual Slice Finish ( std : : unique_ptr < const char [ ] > * buf ) override {
size_t num_entries = hash_entries_ . size ( ) ; return Finish ( buf , nullptr ) ;
}
virtual Slice Finish ( std : : unique_ptr < const char [ ] > * buf ,
Status * status ) override {
size_t num_entries = hash_entries_info_ . entries . size ( ) ;
size_t len_with_metadata = CalculateSpace ( num_entries ) ; size_t len_with_metadata = CalculateSpace ( num_entries ) ;
std : : unique_ptr < char [ ] > mutable_buf ; std : : unique_ptr < char [ ] > mutable_buf ;
std : : unique_ptr < CacheReservationHandle < CacheEntryRole : : kFilterConstruction > >
final_filter_cache_res_handle ;
len_with_metadata = len_with_metadata =
AllocateMaybeRounding ( len_with_metadata , num_entries , & mutable_buf ) ; AllocateMaybeRounding ( len_with_metadata , num_entries , & mutable_buf ) ;
// Cache reservation for mutable_buf
// Cache reservation for mutable_buf
if ( cache_res_mgr_ ) { if ( cache_res_mgr_ ) {
std : : unique_ptr <
CacheReservationHandle < CacheEntryRole : : kFilterConstruction > >
final_filter_cache_res_handle ;
Status s = Status s =
cache_res_mgr_ cache_res_mgr_
- > MakeCacheReservation < CacheEntryRole : : kFilterConstruction > ( - > MakeCacheReservation < CacheEntryRole : : kFilterConstruction > (
len_with_metadata * sizeof ( char ) , len_with_metadata * sizeof ( char ) ,
& final_filter_cache_res_handle ) ; & final_filter_cache_res_handle ) ;
final_filter_cache_res_handles_ . push_back (
std : : move ( final_filter_cache_res_handle ) ) ;
s . PermitUncheckedError ( ) ; s . PermitUncheckedError ( ) ;
} }
@ -282,12 +355,25 @@ class FastLocalBloomBitsBuilder : public XXPH3FilterBitsBuilder {
uint32_t len = static_cast < uint32_t > ( len_with_metadata - kMetadataLen ) ; uint32_t len = static_cast < uint32_t > ( len_with_metadata - kMetadataLen ) ;
if ( len > 0 ) { if ( len > 0 ) {
TEST_SYNC_POINT_CALLBACK (
" XXPH3FilterBitsBuilder::Finish:: "
" TamperHashEntries " ,
& hash_entries_info_ . entries ) ;
AddAllEntries ( mutable_buf . get ( ) , len , num_probes ) ; AddAllEntries ( mutable_buf . get ( ) , len , num_probes ) ;
Status verify_hash_entries_checksum_status =
MaybeVerifyHashEntriesChecksum ( ) ;
if ( ! verify_hash_entries_checksum_status . ok ( ) ) {
if ( status ) {
* status = verify_hash_entries_checksum_status ;
}
return FinishAlwaysTrue ( buf ) ;
}
} }
assert ( hash_entries_ . empty ( ) ) ; bool keep_entries_for_postverify = detect_filter_construct_corruption_ ;
// Release cache for hash entries
if ( ! keep_entries_for_postverify ) {
hash_entry_cache_res_bucket_handles_ . clear ( ) ; ResetEntries ( ) ;
}
// See BloomFilterPolicy::GetBloomBitsReader re: metadata
// See BloomFilterPolicy::GetBloomBitsReader re: metadata
// -1 = Marker for newer Bloom implementations
// -1 = Marker for newer Bloom implementations
@ -298,8 +384,18 @@ class FastLocalBloomBitsBuilder : public XXPH3FilterBitsBuilder {
mutable_buf [ len + 2 ] = static_cast < char > ( num_probes ) ; mutable_buf [ len + 2 ] = static_cast < char > ( num_probes ) ;
// rest of metadata stays zero
// rest of metadata stays zero
auto TEST_arg_pair __attribute__ ( ( __unused__ ) ) =
std : : make_pair ( & mutable_buf , len_with_metadata ) ;
TEST_SYNC_POINT_CALLBACK ( " XXPH3FilterBitsBuilder::Finish::TamperFilter " ,
& TEST_arg_pair ) ;
Slice rv ( mutable_buf . get ( ) , len_with_metadata ) ; Slice rv ( mutable_buf . get ( ) , len_with_metadata ) ;
* buf = std : : move ( mutable_buf ) ; * buf = std : : move ( mutable_buf ) ;
final_filter_cache_res_handles_ . push_back (
std : : move ( final_filter_cache_res_handle ) ) ;
if ( status ) {
* status = Status : : OK ( ) ;
}
return rv ; return rv ;
} }
@ -366,12 +462,12 @@ class FastLocalBloomBitsBuilder : public XXPH3FilterBitsBuilder {
void AddAllEntries ( char * data , uint32_t len , int num_probes ) { void AddAllEntries ( char * data , uint32_t len , int num_probes ) {
// Simple version without prefetching:
// Simple version without prefetching:
//
//
// for (auto h : hash_entries_) {
// for (auto h : hash_entries_info_.entries ) {
// FastLocalBloomImpl::AddHash(Lower32of64(h), Upper32of64(h), len,
// FastLocalBloomImpl::AddHash(Lower32of64(h), Upper32of64(h), len,
// num_probes, data);
// num_probes, data);
// }
// }
const size_t num_entries = hash_entries_ . size ( ) ; const size_t num_entries = hash_entries_info_ . entries . size ( ) ;
constexpr size_t kBufferMask = 7 ; constexpr size_t kBufferMask = 7 ;
static_assert ( ( ( kBufferMask + 1 ) & kBufferMask ) = = 0 , static_assert ( ( ( kBufferMask + 1 ) & kBufferMask ) = = 0 ,
" Must be power of 2 minus 1 " ) ; " Must be power of 2 minus 1 " ) ;
@ -381,12 +477,14 @@ class FastLocalBloomBitsBuilder : public XXPH3FilterBitsBuilder {
// Prime the buffer
// Prime the buffer
size_t i = 0 ; size_t i = 0 ;
std : : deque < uint64_t > : : iterator hash_entries_it =
hash_entries_info_ . entries . begin ( ) ;
for ( ; i < = kBufferMask & & i < num_entries ; + + i ) { for ( ; i < = kBufferMask & & i < num_entries ; + + i ) {
uint64_t h = hash_entries_ . front ( ) ; uint64_t h = * hash_entries_it ;
hash_entries_ . pop_front ( ) ;
FastLocalBloomImpl : : PrepareHash ( Lower32of64 ( h ) , len , data , FastLocalBloomImpl : : PrepareHash ( Lower32of64 ( h ) , len , data ,
/*out*/ & byte_offsets [ i ] ) ; /*out*/ & byte_offsets [ i ] ) ;
hashes [ i ] = Upper32of64 ( h ) ; hashes [ i ] = Upper32of64 ( h ) ;
+ + hash_entries_it ;
} }
// Process and buffer
// Process and buffer
@ -397,11 +495,11 @@ class FastLocalBloomBitsBuilder : public XXPH3FilterBitsBuilder {
FastLocalBloomImpl : : AddHashPrepared ( hash_ref , num_probes , FastLocalBloomImpl : : AddHashPrepared ( hash_ref , num_probes ,
data + byte_offset_ref ) ; data + byte_offset_ref ) ;
// And buffer
// And buffer
uint64_t h = hash_entries_ . front ( ) ; uint64_t h = * hash_entries_it ;
hash_entries_ . pop_front ( ) ;
FastLocalBloomImpl : : PrepareHash ( Lower32of64 ( h ) , len , data , FastLocalBloomImpl : : PrepareHash ( Lower32of64 ( h ) , len , data ,
/*out*/ & byte_offset_ref ) ; /*out*/ & byte_offset_ref ) ;
hash_ref = Upper32of64 ( h ) ; hash_ref = Upper32of64 ( h ) ;
+ + hash_entries_it ;
} }
// Finish processing
// Finish processing
@ -416,7 +514,7 @@ class FastLocalBloomBitsBuilder : public XXPH3FilterBitsBuilder {
} ; } ;
// See description in FastLocalBloomImpl
// See description in FastLocalBloomImpl
class FastLocalBloomBitsReader : public FilterBitsReader { class FastLocalBloomBitsReader : public Builtin FilterBitsReader{
public : public :
FastLocalBloomBitsReader ( const char * data , int num_probes , uint32_t len_bytes ) FastLocalBloomBitsReader ( const char * data , int num_probes , uint32_t len_bytes )
: data_ ( data ) , num_probes_ ( num_probes ) , len_bytes_ ( len_bytes ) { } : data_ ( data ) , num_probes_ ( num_probes ) , len_bytes_ ( len_bytes ) { }
@ -451,6 +549,11 @@ class FastLocalBloomBitsReader : public FilterBitsReader {
} }
} }
bool HashMayMatch ( const uint64_t h ) override {
return FastLocalBloomImpl : : HashMayMatch ( Lower32of64 ( h ) , Upper32of64 ( h ) ,
len_bytes_ , num_probes_ , data_ ) ;
}
private : private :
const char * data_ ; const char * data_ ;
const int num_probes_ ; const int num_probes_ ;
@ -486,12 +589,14 @@ class Standard128RibbonBitsBuilder : public XXPH3FilterBitsBuilder {
explicit Standard128RibbonBitsBuilder ( explicit Standard128RibbonBitsBuilder (
double desired_one_in_fp_rate , int bloom_millibits_per_key , double desired_one_in_fp_rate , int bloom_millibits_per_key ,
std : : atomic < int64_t > * aggregate_rounding_balance , std : : atomic < int64_t > * aggregate_rounding_balance ,
std : : shared_ptr < CacheReservationManager > cache_res_mgr , Logger * info_log ) std : : shared_ptr < CacheReservationManager > cache_res_mgr ,
: XXPH3FilterBitsBuilder ( aggregate_rounding_balance , cache_res_mgr ) , bool detect_filter_construct_corruption , Logger * info_log )
: XXPH3FilterBitsBuilder ( aggregate_rounding_balance , cache_res_mgr ,
detect_filter_construct_corruption ) ,
desired_one_in_fp_rate_ ( desired_one_in_fp_rate ) , desired_one_in_fp_rate_ ( desired_one_in_fp_rate ) ,
info_log_ ( info_log ) , info_log_ ( info_log ) ,
bloom_fallback_ ( bloom_millibits_per_key , aggregate_rounding_balance , bloom_fallback_ ( bloom_millibits_per_key , aggregate_rounding_balance ,
cache_res_mgr ) { cache_res_mgr , detect_filter_construct_corruption ) {
assert ( desired_one_in_fp_rate > = 1.0 ) ; assert ( desired_one_in_fp_rate > = 1.0 ) ;
} }
@ -501,20 +606,32 @@ class Standard128RibbonBitsBuilder : public XXPH3FilterBitsBuilder {
~ Standard128RibbonBitsBuilder ( ) override { } ~ Standard128RibbonBitsBuilder ( ) override { }
using FilterBitsBuilder : : Finish ;
virtual Slice Finish ( std : : unique_ptr < const char [ ] > * buf ) override { virtual Slice Finish ( std : : unique_ptr < const char [ ] > * buf ) override {
if ( hash_entries_ . size ( ) > kMaxRibbonEntries ) { return Finish ( buf , nullptr ) ;
ROCKS_LOG_WARN ( info_log_ , " Too many keys for Ribbon filter: %llu " , }
static_cast < unsigned long long > ( hash_entries_ . size ( ) ) ) ;
virtual Slice Finish ( std : : unique_ptr < const char [ ] > * buf ,
Status * status ) override {
if ( hash_entries_info_ . entries . size ( ) > kMaxRibbonEntries ) {
ROCKS_LOG_WARN (
info_log_ , " Too many keys for Ribbon filter: %llu " ,
static_cast < unsigned long long > ( hash_entries_info_ . entries . size ( ) ) ) ;
SwapEntriesWith ( & bloom_fallback_ ) ; SwapEntriesWith ( & bloom_fallback_ ) ;
assert ( hash_entries_ . empty ( ) ) ; assert ( hash_entries_info_ . entries . empty ( ) ) ;
return bloom_fallback_ . Finish ( buf ) ; return bloom_fallback_ . Finish ( buf , status ) ;
} }
if ( hash_entries_ . size ( ) = = 0 ) { if ( hash_entries_info_ . entries . size ( ) = = 0 ) {
// Save a conditional in Ribbon queries by using alternate reader
// Save a conditional in Ribbon queries by using alternate reader
// for zero entries added.
// for zero entries added.
if ( status ) {
* status = Status : : OK ( ) ;
}
return FinishAlwaysFalse ( buf ) ; return FinishAlwaysFalse ( buf ) ;
} }
uint32_t num_entries = static_cast < uint32_t > ( hash_entries_ . size ( ) ) ; uint32_t num_entries =
static_cast < uint32_t > ( hash_entries_info_ . entries . size ( ) ) ;
uint32_t num_slots ; uint32_t num_slots ;
size_t len_with_metadata ; size_t len_with_metadata ;
@ -523,13 +640,13 @@ class Standard128RibbonBitsBuilder : public XXPH3FilterBitsBuilder {
// Bloom fall-back indicator
// Bloom fall-back indicator
if ( num_slots = = 0 ) { if ( num_slots = = 0 ) {
SwapEntriesWith ( & bloom_fallback_ ) ; SwapEntriesWith ( & bloom_fallback_ ) ;
assert ( hash_entries_ . empty ( ) ) ; assert ( hash_entries_info_ . entries . empty ( ) ) ;
return bloom_fallback_ . Finish ( buf ) ; return bloom_fallback_ . Finish ( buf , status ) ;
} }
uint32_t entropy = 0 ; uint32_t entropy = 0 ;
if ( ! hash_entries_ . empty ( ) ) { if ( ! hash_entries_info_ . entries . empty ( ) ) {
entropy = Lower32of64 ( hash_entries_ . front ( ) ) ; entropy = Lower32of64 ( hash_entries_info_ . entries . front ( ) ) ;
} }
BandingType banding ; BandingType banding ;
@ -552,46 +669,62 @@ class Standard128RibbonBitsBuilder : public XXPH3FilterBitsBuilder {
" Cache reservation for Ribbon filter banding failed due " " Cache reservation for Ribbon filter banding failed due "
" to cache full " ) ; " to cache full " ) ;
SwapEntriesWith ( & bloom_fallback_ ) ; SwapEntriesWith ( & bloom_fallback_ ) ;
assert ( hash_entries_ . empty ( ) ) ; assert ( hash_entries_info_ . entries . empty ( ) ) ;
// Release cache for banding since the banding won't be allocated
// Release cache for banding since the banding won't be allocated
banding_res_handle . reset ( ) ; banding_res_handle . reset ( ) ;
return bloom_fallback_ . Finish ( buf ) ; return bloom_fallback_ . Finish ( buf , status ) ;
} }
TEST_SYNC_POINT_CALLBACK (
" XXPH3FilterBitsBuilder::Finish:: "
" TamperHashEntries " ,
& hash_entries_info_ . entries ) ;
bool success = banding . ResetAndFindSeedToSolve ( bool success = banding . ResetAndFindSeedToSolve (
num_slots , hash_entries_ . begin ( ) , hash_entries_ . end ( ) , num_slots , hash_entries_info_ . entries . begin ( ) ,
hash_entries_info_ . entries . end ( ) ,
/*starting seed*/ entropy & 255 , /*seed mask*/ 255 ) ; /*starting seed*/ entropy & 255 , /*seed mask*/ 255 ) ;
if ( ! success ) { if ( ! success ) {
ROCKS_LOG_WARN ( info_log_ , ROCKS_LOG_WARN (
" Too many re-seeds (256) for Ribbon filter, %llu / %llu " , info_log_ , " Too many re-seeds (256) for Ribbon filter, %llu / %llu " ,
static_cast < unsigned long long > ( hash_entries_ . size ( ) ) , static_cast < unsigned long long > ( hash_entries_info_ . entries . size ( ) ) ,
static_cast < unsigned long long > ( num_slots ) ) ; static_cast < unsigned long long > ( num_slots ) ) ;
SwapEntriesWith ( & bloom_fallback_ ) ; SwapEntriesWith ( & bloom_fallback_ ) ;
assert ( hash_entries_ . empty ( ) ) ; assert ( hash_entries_info_ . entries . empty ( ) ) ;
return bloom_fallback_ . Finish ( buf ) ; return bloom_fallback_ . Finish ( buf , status ) ;
}
Status verify_hash_entries_checksum_status =
MaybeVerifyHashEntriesChecksum ( ) ;
if ( ! verify_hash_entries_checksum_status . ok ( ) ) {
ROCKS_LOG_WARN ( info_log_ , " Verify hash entries checksum error: %s " ,
verify_hash_entries_checksum_status . getState ( ) ) ;
if ( status ) {
* status = verify_hash_entries_checksum_status ;
}
return FinishAlwaysTrue ( buf ) ;
}
bool keep_entries_for_postverify = detect_filter_construct_corruption_ ;
if ( ! keep_entries_for_postverify ) {
ResetEntries ( ) ;
} }
hash_entries_ . clear ( ) ;
// Release cache for hash entries
hash_entry_cache_res_bucket_handles_ . clear ( ) ;
uint32_t seed = banding . GetOrdinalSeed ( ) ; uint32_t seed = banding . GetOrdinalSeed ( ) ;
assert ( seed < 256 ) ; assert ( seed < 256 ) ;
std : : unique_ptr < char [ ] > mutable_buf ; std : : unique_ptr < char [ ] > mutable_buf ;
std : : unique_ptr < CacheReservationHandle < CacheEntryRole : : kFilterConstruction > >
final_filter_cache_res_handle ;
len_with_metadata = len_with_metadata =
AllocateMaybeRounding ( len_with_metadata , num_entries , & mutable_buf ) ; AllocateMaybeRounding ( len_with_metadata , num_entries , & mutable_buf ) ;
// Cache reservation for mutable_buf
// Cache reservation for mutable_buf
if ( cache_res_mgr_ ) { if ( cache_res_mgr_ ) {
std : : unique_ptr <
CacheReservationHandle < CacheEntryRole : : kFilterConstruction > >
final_filter_cache_res_handle ;
Status s = Status s =
cache_res_mgr_ cache_res_mgr_
- > MakeCacheReservation < CacheEntryRole : : kFilterConstruction > ( - > MakeCacheReservation < CacheEntryRole : : kFilterConstruction > (
len_with_metadata * sizeof ( char ) , len_with_metadata * sizeof ( char ) ,
& final_filter_cache_res_handle ) ; & final_filter_cache_res_handle ) ;
final_filter_cache_res_handles_ . push_back (
std : : move ( final_filter_cache_res_handle ) ) ;
s . PermitUncheckedError ( ) ; s . PermitUncheckedError ( ) ;
} }
@ -619,8 +752,18 @@ class Standard128RibbonBitsBuilder : public XXPH3FilterBitsBuilder {
mutable_buf [ len_with_metadata - 1 ] = mutable_buf [ len_with_metadata - 1 ] =
static_cast < char > ( ( num_blocks > > 16 ) & 255 ) ; static_cast < char > ( ( num_blocks > > 16 ) & 255 ) ;
auto TEST_arg_pair __attribute__ ( ( __unused__ ) ) =
std : : make_pair ( & mutable_buf , len_with_metadata ) ;
TEST_SYNC_POINT_CALLBACK ( " XXPH3FilterBitsBuilder::Finish::TamperFilter " ,
& TEST_arg_pair ) ;
Slice rv ( mutable_buf . get ( ) , len_with_metadata ) ; Slice rv ( mutable_buf . get ( ) , len_with_metadata ) ;
* buf = std : : move ( mutable_buf ) ; * buf = std : : move ( mutable_buf ) ;
final_filter_cache_res_handles_ . push_back (
std : : move ( final_filter_cache_res_handle ) ) ;
if ( status ) {
* status = Status : : OK ( ) ;
}
return rv ; return rv ;
} }
@ -637,8 +780,8 @@ class Standard128RibbonBitsBuilder : public XXPH3FilterBitsBuilder {
return ; return ;
} }
uint32_t entropy = 0 ; uint32_t entropy = 0 ;
if ( ! hash_entries_ . empty ( ) ) { if ( ! hash_entries_info_ . entries . empty ( ) ) {
entropy = Upper32of64 ( hash_entries_ . front ( ) ) ; entropy = Upper32of64 ( hash_entries_info_ . entries . front ( ) ) ;
} }
* num_slots = NumEntriesToNumSlots ( static_cast < uint32_t > ( num_entries ) ) ; * num_slots = NumEntriesToNumSlots ( static_cast < uint32_t > ( num_entries ) ) ;
@ -764,6 +907,12 @@ class Standard128RibbonBitsBuilder : public XXPH3FilterBitsBuilder {
return fake_soln . ExpectedFpRate ( ) ; return fake_soln . ExpectedFpRate ( ) ;
} }
Status MaybePostVerify ( const Slice & filter_content ) override {
bool fall_back = ( bloom_fallback_ . EstimateEntriesAdded ( ) > 0 ) ;
return fall_back ? bloom_fallback_ . MaybePostVerify ( filter_content )
: XXPH3FilterBitsBuilder : : MaybePostVerify ( filter_content ) ;
}
protected : protected :
size_t RoundDownUsableSpace ( size_t available_size ) override { size_t RoundDownUsableSpace ( size_t available_size ) override {
size_t rv = available_size - kMetadataLen ; size_t rv = available_size - kMetadataLen ;
@ -808,7 +957,7 @@ class Standard128RibbonBitsBuilder : public XXPH3FilterBitsBuilder {
// for the linker, at least with DEBUG_LEVEL=2
// for the linker, at least with DEBUG_LEVEL=2
constexpr uint32_t Standard128RibbonBitsBuilder : : kMaxRibbonEntries ; constexpr uint32_t Standard128RibbonBitsBuilder : : kMaxRibbonEntries ;
class Standard128RibbonBitsReader : public FilterBitsReader { class Standard128RibbonBitsReader : public Builtin FilterBitsReader{
public : public :
Standard128RibbonBitsReader ( const char * data , size_t len_bytes , Standard128RibbonBitsReader ( const char * data , size_t len_bytes ,
uint32_t num_blocks , uint32_t seed ) uint32_t num_blocks , uint32_t seed )
@ -848,6 +997,10 @@ class Standard128RibbonBitsReader : public FilterBitsReader {
} }
} }
bool HashMayMatch ( const uint64_t h ) override {
return soln_ . FilterQuery ( h , hasher_ ) ;
}
private : private :
using TS = Standard128RibbonTypesAndSettings ; using TS = Standard128RibbonTypesAndSettings ;
ribbon : : SerializableInterleavedSolution < TS > soln_ ; ribbon : : SerializableInterleavedSolution < TS > soln_ ;
@ -874,6 +1027,8 @@ class LegacyBloomBitsBuilder : public BuiltinFilterBitsBuilder {
return hash_entries_ . size ( ) ; return hash_entries_ . size ( ) ;
} }
using FilterBitsBuilder : : Finish ;
Slice Finish ( std : : unique_ptr < const char [ ] > * buf ) override ; Slice Finish ( std : : unique_ptr < const char [ ] > * buf ) override ;
size_t CalculateSpace ( size_t num_entries ) override { size_t CalculateSpace ( size_t num_entries ) override {
@ -1055,7 +1210,7 @@ inline void LegacyBloomBitsBuilder::AddHash(uint32_t h, char* data,
folly : : constexpr_log2 ( CACHE_LINE_SIZE ) ) ; folly : : constexpr_log2 ( CACHE_LINE_SIZE ) ) ;
} }
class LegacyBloomBitsReader : public FilterBitsReader { class LegacyBloomBitsReader : public Builtin FilterBitsReader{
public : public :
LegacyBloomBitsReader ( const char * data , int num_probes , uint32_t num_lines , LegacyBloomBitsReader ( const char * data , int num_probes , uint32_t num_lines ,
uint32_t log2_cache_line_size ) uint32_t log2_cache_line_size )
@ -1100,6 +1255,8 @@ class LegacyBloomBitsReader : public FilterBitsReader {
} }
} }
bool HashMayMatch ( const uint64_t /* h */ ) override { return false ; }
private : private :
const char * data_ ; const char * data_ ;
const int num_probes_ ; const int num_probes_ ;
@ -1107,18 +1264,47 @@ class LegacyBloomBitsReader : public FilterBitsReader {
const uint32_t log2_cache_line_size_ ; const uint32_t log2_cache_line_size_ ;
} ; } ;
class AlwaysTrueFilter : public FilterBitsReader { class AlwaysTrueFilter : public Builtin FilterBitsReader{
public : public :
bool MayMatch ( const Slice & ) override { return true ; } bool MayMatch ( const Slice & ) override { return true ; }
using FilterBitsReader : : MayMatch ; // inherit overload
using FilterBitsReader : : MayMatch ; // inherit overload
bool HashMayMatch ( const uint64_t ) override { return true ; }
using BuiltinFilterBitsReader : : HashMayMatch ; // inherit overload
} ; } ;
class AlwaysFalseFilter : public FilterBitsReader { class AlwaysFalseFilter : public Builtin FilterBitsReader{
public : public :
bool MayMatch ( const Slice & ) override { return false ; } bool MayMatch ( const Slice & ) override { return false ; }
using FilterBitsReader : : MayMatch ; // inherit overload
using FilterBitsReader : : MayMatch ; // inherit overload
bool HashMayMatch ( const uint64_t ) override { return false ; }
using BuiltinFilterBitsReader : : HashMayMatch ; // inherit overload
} ; } ;
Status XXPH3FilterBitsBuilder : : MaybePostVerify ( const Slice & filter_content ) {
Status s = Status : : OK ( ) ;
if ( ! detect_filter_construct_corruption_ ) {
return s ;
}
std : : unique_ptr < BuiltinFilterBitsReader > bits_reader (
BuiltinFilterPolicy : : GetBuiltinFilterBitsReader ( filter_content ) ) ;
for ( uint64_t h : hash_entries_info_ . entries ) {
// The current approach will not detect corruption from XXPH3Filter to
// AlwaysTrueFilter, which can lead to performance cost later due to
// AlwaysTrueFilter not filtering anything. But this cost is acceptable
// given the extra implementation complixity to detect such case.
bool may_match = bits_reader - > HashMayMatch ( h ) ;
if ( ! may_match ) {
s = Status : : Corruption ( " Corrupted filter content " ) ;
break ;
}
}
ResetEntries ( ) ;
return s ;
}
} // namespace
} // namespace
const std : : vector < BloomFilterPolicy : : Mode > BloomFilterPolicy : : kAllFixedImpls = { const std : : vector < BloomFilterPolicy : : Mode > BloomFilterPolicy : : kAllFixedImpls = {
@ -1268,7 +1454,8 @@ FilterBitsBuilder* BloomFilterPolicy::GetBuilderWithContext(
case kFastLocalBloom : case kFastLocalBloom :
return new FastLocalBloomBitsBuilder ( return new FastLocalBloomBitsBuilder (
millibits_per_key_ , offm ? & aggregate_rounding_balance_ : nullptr , millibits_per_key_ , offm ? & aggregate_rounding_balance_ : nullptr ,
cache_res_mgr ) ; cache_res_mgr ,
context . table_options . detect_filter_construct_corruption ) ;
case kLegacyBloom : case kLegacyBloom :
if ( whole_bits_per_key_ > = 14 & & context . info_log & & if ( whole_bits_per_key_ > = 14 & & context . info_log & &
! warned_ . load ( std : : memory_order_relaxed ) ) { ! warned_ . load ( std : : memory_order_relaxed ) ) {
@ -1294,6 +1481,7 @@ FilterBitsBuilder* BloomFilterPolicy::GetBuilderWithContext(
return new Standard128RibbonBitsBuilder ( return new Standard128RibbonBitsBuilder (
desired_one_in_fp_rate_ , millibits_per_key_ , desired_one_in_fp_rate_ , millibits_per_key_ ,
offm ? & aggregate_rounding_balance_ : nullptr , cache_res_mgr , offm ? & aggregate_rounding_balance_ : nullptr , cache_res_mgr ,
context . table_options . detect_filter_construct_corruption ,
context . info_log ) ; context . info_log ) ;
} }
} }
@ -1310,10 +1498,8 @@ FilterBitsBuilder* BloomFilterPolicy::GetBuilderFromContext(
} }
} }
// Read metadata to determine what kind of FilterBitsReader is needed
BuiltinFilterBitsReader * BuiltinFilterPolicy : : GetBuiltinFilterBitsReader (
// and return a new one.
const Slice & contents ) {
FilterBitsReader * BuiltinFilterPolicy : : GetFilterBitsReader (
const Slice & contents ) const {
uint32_t len_with_meta = static_cast < uint32_t > ( contents . size ( ) ) ; uint32_t len_with_meta = static_cast < uint32_t > ( contents . size ( ) ) ;
if ( len_with_meta < = kMetadataLen ) { if ( len_with_meta < = kMetadataLen ) {
// filter is empty or broken. Treat like zero keys added.
// filter is empty or broken. Treat like zero keys added.
@ -1393,8 +1579,15 @@ FilterBitsReader* BuiltinFilterPolicy::GetFilterBitsReader(
log2_cache_line_size ) ; log2_cache_line_size ) ;
} }
FilterBitsReader * BuiltinFilterPolicy : : GetRibbonBitsReader ( // Read metadata to determine what kind of FilterBitsReader is needed
// and return a new one.
FilterBitsReader * BuiltinFilterPolicy : : GetFilterBitsReader (
const Slice & contents ) const { const Slice & contents ) const {
return BuiltinFilterPolicy : : GetBuiltinFilterBitsReader ( contents ) ;
}
BuiltinFilterBitsReader * BuiltinFilterPolicy : : GetRibbonBitsReader (
const Slice & contents ) {
uint32_t len_with_meta = static_cast < uint32_t > ( contents . size ( ) ) ; uint32_t len_with_meta = static_cast < uint32_t > ( contents . size ( ) ) ;
uint32_t len = len_with_meta - kMetadataLen ; uint32_t len = len_with_meta - kMetadataLen ;
@ -1417,8 +1610,8 @@ FilterBitsReader* BuiltinFilterPolicy::GetRibbonBitsReader(
} }
// For newer Bloom filter implementations
// For newer Bloom filter implementations
FilterBitsReader * BuiltinFilterPolicy : : GetBloomBitsReader ( Builtin FilterBitsReader* BuiltinFilterPolicy : : GetBloomBitsReader (
const Slice & contents ) const { const Slice & contents ) {
uint32_t len_with_meta = static_cast < uint32_t > ( contents . size ( ) ) ; uint32_t len_with_meta = static_cast < uint32_t > ( contents . size ( ) ) ;
uint32_t len = len_with_meta - kMetadataLen ; uint32_t len = len_with_meta - kMetadataLen ;
Hui Xiao
3 years ago
committed by