From 130e710056f48d5b2da64b218926319ce9f3fe41 Mon Sep 17 00:00:00 2001 From: Levi Tamasi Date: Wed, 18 Dec 2019 17:44:29 -0800 Subject: [PATCH] Add BlobDB GC cutoff parameter to db_bench (#6211) Summary: The patch makes it possible to set the BlobDB configuration option `garbage_collection_cutoff` on the command line. In addition, it changes the `db_bench` code so that the default values of BlobDB related parameters are taken from the defaults of the actual BlobDB configuration options (note: this changes the the default of `blob_db_bytes_per_sync`). Pull Request resolved: https://github.com/facebook/rocksdb/pull/6211 Test Plan: Ran `db_bench` with various values of the new parameter. Differential Revision: D19166895 Pulled By: ltamasi fbshipit-source-id: 305ccdf0123b9db032b744715810babdc3e3b7d5 --- HISTORY.md | 1 + tools/db_bench_tool.cc | 28 +++++++++++++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 626da9908..0f35a44d9 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -15,6 +15,7 @@ ### New Features * It is now possible to enable periodic compactions for the base DB when using BlobDB. * BlobDB now garbage collects non-TTL blobs when `enable_garbage_collection` is set to `true` in `BlobDBOptions`. Garbage collection is performed during compaction: any valid blobs located in the oldest N files (where N is the number of non-TTL blob files multiplied by the value of `BlobDBOptions::garbage_collection_cutoff`) encountered during compaction get relocated to new blob files, and old blob files are dropped once they are no longer needed. Note: we recommend enabling periodic compactions for the base DB when using this feature to deal with the case when some old blob files are kept alive by SSTs that otherwise do not get picked for compaction. +* `db_bench` now supports the `garbage_collection_cutoff` option for BlobDB. ## 6.6.0 (11/25/2019) ### Bug Fixes diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index d0540cd0c..383a12062 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -750,27 +750,40 @@ DEFINE_bool(use_blob_db, false, "Open a BlobDB instance. " "Required for large value benchmark."); -DEFINE_bool(blob_db_enable_gc, false, "Enable BlobDB garbage collection."); +DEFINE_bool(blob_db_enable_gc, + rocksdb::blob_db::BlobDBOptions().enable_garbage_collection, + "Enable BlobDB garbage collection."); -DEFINE_bool(blob_db_is_fifo, false, "Enable FIFO eviction strategy in BlobDB."); +DEFINE_double(blob_db_gc_cutoff, + rocksdb::blob_db::BlobDBOptions().garbage_collection_cutoff, + "Cutoff ratio for BlobDB garbage collection."); -DEFINE_uint64(blob_db_max_db_size, 0, +DEFINE_bool(blob_db_is_fifo, rocksdb::blob_db::BlobDBOptions().is_fifo, + "Enable FIFO eviction strategy in BlobDB."); + +DEFINE_uint64(blob_db_max_db_size, + rocksdb::blob_db::BlobDBOptions().max_db_size, "Max size limit of the directory where blob files are stored."); DEFINE_uint64( blob_db_max_ttl_range, 0, "TTL range to generate BlobDB data (in seconds). 0 means no TTL."); -DEFINE_uint64(blob_db_ttl_range_secs, 3600, +DEFINE_uint64(blob_db_ttl_range_secs, + rocksdb::blob_db::BlobDBOptions().ttl_range_secs, "TTL bucket size to use when creating blob files."); -DEFINE_uint64(blob_db_min_blob_size, 0, +DEFINE_uint64(blob_db_min_blob_size, + rocksdb::blob_db::BlobDBOptions().min_blob_size, "Smallest blob to store in a file. Blobs smaller than this " "will be inlined with the key in the LSM tree."); -DEFINE_uint64(blob_db_bytes_per_sync, 0, "Bytes to sync blob file at."); +DEFINE_uint64(blob_db_bytes_per_sync, + rocksdb::blob_db::BlobDBOptions().bytes_per_sync, + "Bytes to sync blob file at."); -DEFINE_uint64(blob_db_file_size, 256 * 1024 * 1024, +DEFINE_uint64(blob_db_file_size, + rocksdb::blob_db::BlobDBOptions().blob_file_size, "Target size of each blob file."); // Secondary DB instance Options @@ -3994,6 +4007,7 @@ class Benchmark { } else if (FLAGS_use_blob_db) { blob_db::BlobDBOptions blob_db_options; blob_db_options.enable_garbage_collection = FLAGS_blob_db_enable_gc; + blob_db_options.garbage_collection_cutoff = FLAGS_blob_db_gc_cutoff; blob_db_options.is_fifo = FLAGS_blob_db_is_fifo; blob_db_options.max_db_size = FLAGS_blob_db_max_db_size; blob_db_options.ttl_range_secs = FLAGS_blob_db_ttl_range_secs;