NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support ZSTD with finalized format

Browse Source 
Summary:
ZSTD 1.0.0 is coming. We can finally add a support of ZSTD without worrying about compatibility.
Still keep ZSTDNotFinal for compatibility reason.

Test Plan: Run all tests. Run db_bench with ZSTD version with RocksDB built with ZSTD 1.0 and older.

Reviewers: andrewkr, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: cyan, igor, IslamAbdelRahman, leveldb, andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D63141
main
sdong 8 years ago
parent ce1be2ce37
commit 607628d349
18 changed files with 51 additions and 30 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      HISTORY.md
  2. 2
      build_tools/dependencies.sh
  3. 2
      build_tools/dependencies_4.8.1.sh
  4. 4
      db/db_test.cc
  5. 6
      db/db_test2.cc
  6. 8
      include/rocksdb/options.h
  7. 1
      table/block_based_table_builder.cc
  8. 1
      table/format.cc
  9. 4
      table/table_test.cc
  10. 6
      tools/db_bench_tool.cc
  11. 2
      tools/db_sanity_test.cc
  12. 2
      tools/db_stress.cc
  13. 2
      tools/ldb_cmd.cc
  14. 19
      tools/sst_dump_tool.cc
  15. 11
      util/compression.h
  16. 1
      util/options_helper.h
  17. 6
      util/options_test.cc
  18. 3
      utilities/column_aware_encoding_exp.cc

1
HISTORY.md
Unescape View File

@ -4,6 +4,7 @@
* CancelAllBackgroundWork() flushes all memtables for databases containing writes that have bypassed the WAL (writes issued with WriteOptions::disableWAL=true) before shutting down background threads.
* Merge options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor into max_compaction_bytes.
* Remove ImmutableCFOptions.
* Add a compression type ZSTD, which can work with ZSTD 0.8.0 or up. Still keep ZSTDNotFinal for compatibility reasons.
### New Features
* Introduce NewClockCache, which is based on CLOCK algorithm with better concurrent performance in some cases. It can be used to replace the default LRU-based block cache and table cache. To use it, RocksDB need to be linked with TBB lib.

2
build_tools/dependencies.sh
Unescape View File

@ -6,7 +6,7 @@ SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfc
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-5-glibc-2.23/9bc6787
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-5-glibc-2.23/9bc6787
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-5-glibc-2.23/9bc6787
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/bd13f49da8633897105b2b8541b106d487ce54ca/0.7.5/gcc-5-glibc-2.23/9bc6787
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-5-glibc-2.23/9bc6787
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-5-glibc-2.23/1c32b4b
NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-5-glibc-2.23/9bc6787

2
build_tools/dependencies_4.8.1.sh
Unescape View File

@ -6,7 +6,7 @@ SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfc
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-4.8.1-glibc-2.17/c3f970a
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/bd13f49da8633897105b2b8541b106d487ce54ca/0.7.5/gcc-4.8.1-glibc-2.17/c3f970a
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-4.8.1-glibc-2.17/c3f970a
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-4.8.1-glibc-2.17/8d31e51
NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a

4
db/db_test.cc
Unescape View File

@ -1193,7 +1193,7 @@ bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
type = kXpressCompression;
fprintf(stderr, "using xpress\n");
} else if (ZSTD_Supported()) {
type = kZSTDNotFinalCompression;
type = kZSTD;
fprintf(stderr, "using ZSTD\n");
} else {
fprintf(stderr, "skipping test, compression disabled\n");
@ -4708,7 +4708,7 @@ TEST_F(DBTest, CompressionStatsTest) {
type = kXpressCompression;
fprintf(stderr, "using xpress\n");
} else if (ZSTD_Supported()) {
type = kZSTDNotFinalCompression;
type = kZSTD;
fprintf(stderr, "using ZSTD\n");
} else {
fprintf(stderr, "skipping test, compression disabled\n");

6
db/db_test2.cc
Unescape View File

@ -961,9 +961,9 @@ TEST_F(DBTest2, PresetCompressionDict) {
compression_types.push_back(kLZ4Compression);
compression_types.push_back(kLZ4HCCompression);
#endif // LZ4_VERSION_NUMBER >= 10400
#if ZSTD_VERSION_NUMBER >= 500 // v0.5.0+
compression_types.push_back(kZSTDNotFinalCompression);
#endif // ZSTD_VERSION_NUMBER >= 500
if (ZSTD_Supported()) {
compression_types.push_back(kZSTD);
}
for (auto compression_type : compression_types) {
options.compression = compression_type;

8
include/rocksdb/options.h
Unescape View File

@ -63,7 +63,13 @@ enum CompressionType : unsigned char {
kLZ4Compression = 0x4,
kLZ4HCCompression = 0x5,
kXpressCompression = 0x6,
// zstd format is not finalized yet so it's subject to changes.
kZSTD = 0x7,
// Only use kZSTDNotFinalCompression if you have to use ZSTD lib older than
// 0.8.0 or consider a possibility of downgrading the service or copying
// the database files to another service running with an older version of
// RocksDB that doesn't have kZSTD. Otherwise, you should use kZSTD. We will
// eventually remove the option from the public API.
kZSTDNotFinalCompression = 0x40,
// kDisableCompressionOption is used to disable some compression options.

1
table/block_based_table_builder.cc
Unescape View File

@ -377,6 +377,7 @@ Slice CompressBlock(const Slice& raw,
return *compressed_output;
}
break;
case kZSTD:
case kZSTDNotFinalCompression:
if (ZSTD_Compress(compression_options, raw.data(), raw.size(),
compressed_output, compression_dict) &&

1
table/format.cc
Unescape View File

@ -498,6 +498,7 @@ Status UncompressBlockContentsForCompressionType(
*contents =
BlockContents(std::move(ubuf), decompress_size, true, kNoCompression);
break;
case kZSTD:
case kZSTDNotFinalCompression:
ubuf.reset(ZSTD_Uncompress(data, n, &decompress_size, compression_dict));
if (!ubuf) {

4
table/table_test.cc
Unescape View File

@ -588,8 +588,8 @@ static std::vector<TestArgs> GenerateArgList() {
compression_types.emplace_back(kXpressCompression, true);
}
if (ZSTD_Supported()) {
compression_types.emplace_back(kZSTDNotFinalCompression, false);
compression_types.emplace_back(kZSTDNotFinalCompression, true);
compression_types.emplace_back(kZSTD, false);
compression_types.emplace_back(kZSTD, true);
}
for (auto test_type : test_types) {

6
tools/db_bench_tool.cc
Unescape View File

@ -597,7 +597,7 @@ enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
else if (!strcasecmp(ctype, "xpress"))
return rocksdb::kXpressCompression;
else if (!strcasecmp(ctype, "zstd"))
return rocksdb::kZSTDNotFinalCompression;
return rocksdb::kZSTD;
fprintf(stdout, "Cannot parse compression type '%s'\n", ctype);
return rocksdb::kSnappyCompression; // default value
@ -1771,7 +1771,7 @@ class Benchmark {
ok = XPRESS_Compress(input.data(),
input.size(), compressed);
break;
case rocksdb::kZSTDNotFinalCompression:
case rocksdb::kZSTD:
ok = ZSTD_Compress(Options().compression_opts, input.data(),
input.size(), compressed);
break;
@ -2606,7 +2606,7 @@ class Benchmark {
&decompress_size);
ok = uncompressed != nullptr;
break;
case rocksdb::kZSTDNotFinalCompression:
case rocksdb::kZSTD:
uncompressed = ZSTD_Uncompress(compressed.data(), compressed.size(),
&decompress_size);
ok = uncompressed != nullptr;

2
tools/db_sanity_test.cc
Unescape View File

@ -186,7 +186,7 @@ class SanityTestZSTDCompression : public SanityTest {
public:
explicit SanityTestZSTDCompression(const std::string& path)
: SanityTest(path) {
options_.compression = kZSTDNotFinalCompression;
options_.compression = kZSTD;
}
virtual Options GetOptions() const override { return options_; }
virtual std::string Name() const override { return "ZSTDCompression"; }

2
tools/db_stress.cc
Unescape View File

@ -375,7 +375,7 @@ enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
else if (!strcasecmp(ctype, "xpress"))
return rocksdb::kXpressCompression;
else if (!strcasecmp(ctype, "zstd"))
return rocksdb::kZSTDNotFinalCompression;
return rocksdb::kZSTD;
fprintf(stdout, "Cannot parse compression type '%s'\n", ctype);
return rocksdb::kSnappyCompression; //default value

2
tools/ldb_cmd.cc
Unescape View File

@ -509,7 +509,7 @@ Options LDBCommand::PrepareOptionsForOpenDB() {
} else if (comp == "xpress") {
opt.compression = kXpressCompression;
} else if (comp == "zstd") {
opt.compression = kZSTDNotFinalCompression;
opt.compression = kZSTD;
} else {
// Unknown compression.
exec_state_ =

19
tools/sst_dump_tool.cc
Unescape View File

@ -185,16 +185,15 @@ int SstFileReader::ShowAllCompressionSizes(size_t block_size) {
fprintf(stdout, "Block Size: %" ROCKSDB_PRIszt "\n", block_size);
std::pair<CompressionType,const char*> compressions[] = {
{ CompressionType::kNoCompression, "kNoCompression" },
{ CompressionType::kSnappyCompression, "kSnappyCompression" },
{ CompressionType::kZlibCompression, "kZlibCompression" },
{ CompressionType::kBZip2Compression, "kBZip2Compression" },
{ CompressionType::kLZ4Compression, "kLZ4Compression" },
{ CompressionType::kLZ4HCCompression, "kLZ4HCCompression" },
{ CompressionType::kXpressCompression, "kXpressCompression" },
{ CompressionType::kZSTDNotFinalCompression, "kZSTDNotFinalCompression" }
};
std::pair<CompressionType, const char*> compressions[] = {
{CompressionType::kNoCompression, "kNoCompression"},
{CompressionType::kSnappyCompression, "kSnappyCompression"},
{CompressionType::kZlibCompression, "kZlibCompression"},
{CompressionType::kBZip2Compression, "kBZip2Compression"},
{CompressionType::kLZ4Compression, "kLZ4Compression"},
{CompressionType::kLZ4HCCompression, "kLZ4HCCompression"},
{CompressionType::kXpressCompression, "kXpressCompression"},
{CompressionType::kZSTD, "kZSTD"}};
for (auto& i : compressions) {
if (CompressionTypeSupported(i.first)) {

11
util/compression.h
Unescape View File

@ -79,6 +79,14 @@ inline bool XPRESS_Supported() {
}
inline bool ZSTD_Supported() {
#ifdef ZSTD
// ZSTD format is finalized since version 0.8.0.
return (ZSTD_versionNumber() >= 800);
#endif
return false;
}
inline bool ZSTDNotFinal_Supported() {
#ifdef ZSTD
return true;
#endif
@ -102,6 +110,8 @@ inline bool CompressionTypeSupported(CompressionType compression_type) {
case kXpressCompression:
return XPRESS_Supported();
case kZSTDNotFinalCompression:
return ZSTDNotFinal_Supported();
case kZSTD:
return ZSTD_Supported();
default:
assert(false);
@ -125,6 +135,7 @@ inline std::string CompressionTypeToString(CompressionType compression_type) {
return "LZ4HC";
case kXpressCompression:
return "Xpress";
case kZSTD:
case kZSTDNotFinalCompression:
return "ZSTD";
default:

1
util/options_helper.h
Unescape View File

@ -598,6 +598,7 @@ static std::unordered_map<std::string, CompressionType>
{"kLZ4Compression", kLZ4Compression},
{"kLZ4HCCompression", kLZ4HCCompression},
{"kXpressCompression", kXpressCompression},
{"kZSTD", kZSTD},
{"kZSTDNotFinalCompression", kZSTDNotFinalCompression},
{"kDisableCompressionOption", kDisableCompressionOption}};

6
util/options_test.cc
Unescape View File

@ -56,6 +56,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
"kLZ4Compression:"
"kLZ4HCCompression:"
"kXpressCompression:"
"kZSTD:"
"kZSTDNotFinalCompression"},
{"bottommost_compression", "kLZ4Compression"},
{"compression_opts", "4:5:6:7"},
@ -140,7 +141,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3);
ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99);
ASSERT_EQ(new_cf_opt.compression, kSnappyCompression);
ASSERT_EQ(new_cf_opt.compression_per_level.size(), 8U);
ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U);
ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression);
ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression);
ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression);
@ -148,7 +149,8 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression);
ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression);
ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression);
ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTDNotFinalCompression);
ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD);
ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression);
ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4);
ASSERT_EQ(new_cf_opt.compression_opts.level, 5);
ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6);

3
utilities/column_aware_encoding_exp.cc
Unescape View File

@ -80,8 +80,7 @@ class ColumnAwareEncodingExp {
std::unordered_map<std::string, CompressionType> compressions = {
{"kNoCompression", CompressionType::kNoCompression},
{"kZlibCompression", CompressionType::kZlibCompression},
{"kZSTDNotFinalCompression",
CompressionType::kZSTDNotFinalCompression}};
{"kZSTD", CompressionType::kZSTD}};
// Find Compression
CompressionType compression_type = compressions[FLAGS_compression_type];

Write Preview
Loading…
Cancel
Save