NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix compression tests when snappy not available (#11396)

Browse Source 
Summary:
Tweak some bounds and things, and reduce risk of surprise results by running on all supported compressions (mostly).

Also improves the precise compressibility of CompressibleString by using RandomBinaryString.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/11396

Test Plan: updated tests

Reviewed By: ltamasi

Differential Revision: D45211938

Pulled By: pdillinger

fbshipit-source-id: 9dc1dd8574a60a9364efe18558be66d31a35598b
oxigraph-8.3.2
Peter Dillinger 2 years ago committed by Facebook GitHub Bot
parent d79be3dca2
commit fb63d9b4ee
3 changed files with 145 additions and 175 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 226
      db/db_statistics_test.cc
  2. 92
      table/table_test.cc
  3. 2
      test_util/testutil.cc

226
db/db_statistics_test.cc
Unescape View File

@ -20,131 +20,115 @@ class DBStatisticsTest : public DBTestBase {
}; };
TEST_F(DBStatisticsTest, CompressionStatsTest) { TEST_F(DBStatisticsTest, CompressionStatsTest) {
CompressionType type; for (CompressionType type : GetSupportedCompressions()) {
if (type == kNoCompression) {
if (Snappy_Supported()) { continue;
type = kSnappyCompression; }
fprintf(stderr, "using snappy\n"); if (type == kBZip2Compression) {
} else if (Zlib_Supported()) { // Weird behavior in this test
type = kZlibCompression; continue;
fprintf(stderr, "using zlib\n"); }
} else if (BZip2_Supported()) { SCOPED_TRACE("Compression type: " + std::to_string(type));
type = kBZip2Compression;
fprintf(stderr, "using bzip2\n"); Options options = CurrentOptions();
} else if (LZ4_Supported()) { options.compression = type;
type = kLZ4Compression; options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
fprintf(stderr, "using lz4\n"); options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex);
} else if (XPRESS_Supported()) { BlockBasedTableOptions bbto;
type = kXpressCompression; bbto.enable_index_compression = false;
fprintf(stderr, "using xpress\n"); options.table_factory.reset(NewBlockBasedTableFactory(bbto));
} else if (ZSTD_Supported()) { DestroyAndReopen(options);
type = kZSTD;
fprintf(stderr, "using ZSTD\n"); auto PopStat = [&](Tickers t) -> uint64_t {
} else { return options.statistics->getAndResetTickerCount(t);
fprintf(stderr, "skipping test, compression disabled\n"); };
return;
} int kNumKeysWritten = 100;
double compress_to = 0.5;
Options options = CurrentOptions(); // About three KVs per block
options.compression = type; int len = static_cast<int>(BlockBasedTableOptions().block_size / 3);
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); int uncomp_est = kNumKeysWritten * (len + 20);
options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex);
BlockBasedTableOptions bbto; Random rnd(301);
bbto.enable_index_compression = false; std::string buf;
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
DestroyAndReopen(options); // Check that compressions occur and are counted when compression is turned
// on
auto PopStat = [&](Tickers t) -> uint64_t { for (int i = 0; i < kNumKeysWritten; ++i) {
return options.statistics->getAndResetTickerCount(t); ASSERT_OK(
}; Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf)));
}
int kNumKeysWritten = 100; ASSERT_OK(Flush());
double compress_to = 0.5; EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSED));
// About three KVs per block EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSED_FROM), uncomp_est / 10);
int len = static_cast<int>(BlockBasedTableOptions().block_size / 3); EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_COMPRESSED_TO),
int uncomp_est = kNumKeysWritten * (len + 20); uncomp_est / 10);
Random rnd(301); EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
std::string buf; EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
// And decompressions
for (int i = 0; i < kNumKeysWritten; ++i) {
auto r = Get(Key(i));
}
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_DECOMPRESSED));
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_DECOMPRESSED_TO), uncomp_est / 10);
EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_DECOMPRESSED_FROM),
uncomp_est / 10);
// Check that compressions occur and are counted when compression is turned on EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
for (int i = 0; i < kNumKeysWritten; ++i) { EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
ASSERT_OK( EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf))); EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
}
ASSERT_OK(Flush());
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSED));
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSED_FROM), uncomp_est / 10);
EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_COMPRESSED_TO),
uncomp_est / 10);
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
// And decompressions
for (int i = 0; i < kNumKeysWritten; ++i) {
auto r = Get(Key(i));
}
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_DECOMPRESSED));
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_DECOMPRESSED_TO), uncomp_est / 10);
EXPECT_NEAR2(uncomp_est * compress_to, PopStat(BYTES_DECOMPRESSED_FROM),
uncomp_est / 10);
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
// Check when compression is rejected.
compress_to = 0.95;
DestroyAndReopen(options);
for (int i = 0; i < kNumKeysWritten; ++i) { // Check when compression is rejected.
ASSERT_OK( DestroyAndReopen(options);
Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf)));
}
ASSERT_OK(Flush());
for (int i = 0; i < kNumKeysWritten; ++i) {
auto r = Get(Key(i));
}
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_REJECTED),
uncomp_est / 10);
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
// Check when compression is disabled.
options.compression = kNoCompression;
DestroyAndReopen(options);
for (int i = 0; i < kNumKeysWritten; ++i) { for (int i = 0; i < kNumKeysWritten; ++i) {
ASSERT_OK( ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len)));
Put(Key(i), test::CompressibleString(&rnd, compress_to, len, &buf))); }
} ASSERT_OK(Flush());
ASSERT_OK(Flush()); for (int i = 0; i < kNumKeysWritten; ++i) {
for (int i = 0; i < kNumKeysWritten; ++i) { auto r = Get(Key(i));
auto r = Get(Key(i)); }
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_REJECTED),
uncomp_est / 10);
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_BYPASSED));
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
// Check when compression is disabled.
options.compression = kNoCompression;
DestroyAndReopen(options);
for (int i = 0; i < kNumKeysWritten; ++i) {
ASSERT_OK(Put(Key(i), rnd.RandomBinaryString(len)));
}
ASSERT_OK(Flush());
for (int i = 0; i < kNumKeysWritten; ++i) {
auto r = Get(Key(i));
}
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_BYPASSED),
uncomp_est / 10);
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
} }
EXPECT_EQ(34, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
EXPECT_NEAR2(uncomp_est, PopStat(BYTES_COMPRESSION_BYPASSED),
uncomp_est / 10);
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSED));
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
EXPECT_EQ(0, PopStat(NUMBER_BLOCK_DECOMPRESSED));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_FROM));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSED_TO));
EXPECT_EQ(0, PopStat(BYTES_COMPRESSION_REJECTED));
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_FROM));
EXPECT_EQ(0, PopStat(BYTES_DECOMPRESSED_TO));
} }
TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) { TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) {

92
table/table_test.cc
Unescape View File

@ -5081,63 +5081,49 @@ TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) {
} }
TEST_P(BlockBasedTableTest, CompressionRatioThreshold) { TEST_P(BlockBasedTableTest, CompressionRatioThreshold) {
Options options; for (CompressionType type : GetSupportedCompressions()) {
if (Snappy_Supported()) { if (type == kNoCompression) {
options.compression = kSnappyCompression; continue;
fprintf(stderr, "using snappy\n"); }
} else if (Zlib_Supported()) { SCOPED_TRACE("Compression type: " + std::to_string(type));
options.compression = kZlibCompression;
fprintf(stderr, "using zlib\n");
} else if (BZip2_Supported()) {
options.compression = kBZip2Compression;
fprintf(stderr, "using bzip2\n");
} else if (LZ4_Supported()) {
options.compression = kLZ4Compression;
fprintf(stderr, "using lz4\n");
} else if (XPRESS_Supported()) {
options.compression = kXpressCompression;
fprintf(stderr, "using xpress\n");
} else if (ZSTD_Supported()) {
options.compression = kZSTD;
fprintf(stderr, "using ZSTD\n");
} else {
fprintf(stderr, "skipping test, compression disabled\n");
return;
}
BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
int len = 10000;
Random rnd(301);
std::vector<std::string> keys;
stl_wrappers::KVMap kvmap;
// Test the max_compressed_bytes_per_kb option Options options;
for (int threshold : {0, 1, 100, 400, 600, 900, 1024}) { options.compression = type;
SCOPED_TRACE("threshold=" + std::to_string(threshold));
options.compression_opts.max_compressed_bytes_per_kb = threshold;
ImmutableOptions ioptions(options);
MutableCFOptions moptions(options);
for (double compressible_to : {0.25, 0.75}) { BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
SCOPED_TRACE("compressible_to=" + std::to_string(compressible_to)); int len = 10000;
TableConstructor c(BytewiseComparator(), Random rnd(301);
true /* convert_to_internal_key_ */); std::vector<std::string> keys;
std::string buf; stl_wrappers::KVMap kvmap;
c.Add("x", test::CompressibleString(&rnd, compressible_to, len, &buf));
// write an SST file // Test the max_compressed_bytes_per_kb option
c.Finish(options, ioptions, moptions, table_options, for (int threshold : {0, 1, 100, 400, 600, 900, 1024}) {
GetPlainInternalComparator(options.comparator), &keys, &kvmap); SCOPED_TRACE("threshold=" + std::to_string(threshold));
options.compression_opts.max_compressed_bytes_per_kb = threshold;
ImmutableOptions ioptions(options);
MutableCFOptions moptions(options);
size_t table_file_size = c.TEST_GetSink()->contents().size(); for (double compressible_to : {0.25, 0.75}) {
size_t approx_sst_overhead = 1000; SCOPED_TRACE("compressible_to=" + std::to_string(compressible_to));
if (compressible_to < threshold / 1024.0) { TableConstructor c(BytewiseComparator(),
// Should be compressed true /* convert_to_internal_key_ */);
EXPECT_NEAR2(len * compressible_to + approx_sst_overhead, std::string buf;
table_file_size, len / 10); c.Add("x", test::CompressibleString(&rnd, compressible_to, len, &buf));
} else {
// Should not be compressed // write an SST file
EXPECT_NEAR2(len + approx_sst_overhead, table_file_size, len / 10); c.Finish(options, ioptions, moptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
size_t table_file_size = c.TEST_GetSink()->contents().size();
size_t approx_sst_overhead = 1000;
if (compressible_to < threshold / 1024.0) {
// Should be compressed (substantial variance depending on algorithm)
EXPECT_NEAR2(len * compressible_to + approx_sst_overhead,
table_file_size, len / 8);
} else {
// Should not be compressed
EXPECT_NEAR2(len + approx_sst_overhead, table_file_size, len / 10);
}
} }
} }
} }

2
test_util/testutil.cc
Unescape View File

@ -76,7 +76,7 @@ extern Slice CompressibleString(Random* rnd, double compressed_fraction,
int len, std::string* dst) { int len, std::string* dst) {
int raw = static_cast<int>(len * compressed_fraction); int raw = static_cast<int>(len * compressed_fraction);
if (raw < 1) raw = 1; if (raw < 1) raw = 1;
std::string raw_data = rnd->RandomString(raw); std::string raw_data = rnd->RandomBinaryString(raw);
// Duplicate the random data until we have filled "len" bytes // Duplicate the random data until we have filled "len" bytes
dst->clear(); dst->clear();

Write Preview
Loading…
Cancel
Save