From 50046869a466efd19456562dad643951900acbb2 Mon Sep 17 00:00:00 2001 From: Hui Xiao Date: Thu, 18 May 2023 09:26:29 -0700 Subject: [PATCH] Add `rocksdb.file.read.db.open.micros` (#11455) Summary: **Context/Summary:** `rocksdb.file.read.db.open.micros` is left out in https://github.com/facebook/rocksdb/pull/11288 Pull Request resolved: https://github.com/facebook/rocksdb/pull/11455 Test Plan: - db bench Setup: `./db_bench -db=/dev/shm/testdb/ -statistics=true -benchmarks="fillseq" -key_size=32 -value_size=512 -num=5000 -write_buffer_size=655 -target_file_size_base=655 -disable_auto_compactions=false -compression_type=none -bloom_bits=3` Run: `./db_bench --bloom_bits=3 --use_existing_db=1 --seed=1682546046158958 --partition_index_and_filters=1 --statistics=1 -db=/dev/shm/testdb/ -benchmarks=readrandom -key_size=3200 -value_size=512 -num=0 -write_buffer_size=6550000 -disable_auto_compactions=false -target_file_size_base=6550000 -compression_type=none -file_checksum=1 -cache_size=1` ``` rocksdb.sst.read.micros P50 : 3.979798 P95 : 9.738420 P99 : 19.566667 P100 : 39.000000 COUNT : 2360 SUM : 12148 rocksdb.file.read.flush.micros P50 : 0.000000 P95 : 0.000000 P99 : 0.000000 P100 : 0.000000 COUNT : 0 SUM : 0 rocksdb.file.read.compaction.micros P50 : 0.000000 P95 : 0.000000 P99 : 0.000000 P100 : 0.000000 COUNT : 0 SUM : 0 rocksdb.file.read.db.open.micros P50 : 3.979798 P95 : 9.738420 P99 : 19.566667 P100 : 39.000000 COUNT : 2360 SUM : 12148 ``` Reviewed By: ajkr Differential Revision: D45951934 Pulled By: hx235 fbshipit-source-id: 6c88639dc1b10d98ecccc963ce32a8800495f55b --- HISTORY.md | 1 + file/random_access_file_reader.cc | 1 + include/rocksdb/statistics.h | 3 ++- java/rocksjni/portal.h | 4 ++++ java/src/main/java/org/rocksdb/HistogramType.java | 2 ++ monitoring/statistics.cc | 1 + 6 files changed, 11 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 6fa5dc855..d76c99e59 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -6,6 +6,7 @@ * Improve the operational safety of publishing a DB or SST files to many hosts by using different block cache hash seeds on different hosts. The exact behavior is controlled by new option `ShardedCacheOptions::hash_seed`, which also documents the solved problem in more detail. * Introduced a new option `CompactionOptionsFIFO::file_temperature_age_thresholds` that allows FIFO compaction to compact files to different temperatures based on key age (#11428). * Added a new ticker stat to count how many times RocksDB detected a corruption while verifying a block checksum: `BLOCK_CHECKSUM_MISMATCH_COUNT`. +* New statistics `rocksdb.file.read.db.open.micros` that measures read time of block-based SST tables or blob files during db open. ### Public API Changes * Add `MakeSharedCache()` construction functions to various cache Options objects, and deprecated the `NewWhateverCache()` functions with long parameter lists. diff --git a/file/random_access_file_reader.cc b/file/random_access_file_reader.cc index 7a1af1965..308cd3e5b 100644 --- a/file/random_access_file_reader.cc +++ b/file/random_access_file_reader.cc @@ -26,6 +26,7 @@ const std::array kReadHistograms{{ FILE_READ_FLUSH_MICROS, FILE_READ_COMPACTION_MICROS, + FILE_READ_DB_OPEN_MICROS, }}; inline void RecordIOStats(Statistics* stats, Temperature file_temperature, bool is_last_level, size_t size) { diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index 99791e47a..e93e3f160 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -512,9 +512,10 @@ enum Histograms : uint32_t { // Time spent in reading block-based or plain SST table SST_READ_MICROS, // Time spent in reading SST table (currently only block-based table) or blob - // file for flush or compaction + // file corresponding to `Env::IOActivity` FILE_READ_FLUSH_MICROS, FILE_READ_COMPACTION_MICROS, + FILE_READ_DB_OPEN_MICROS, // The number of subcompactions actually scheduled during a compaction NUM_SUBCOMPACTIONS_SCHEDULED, diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index 867477f1a..aa0cc6131 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -5627,6 +5627,8 @@ class HistogramTypeJni { return 0x3A; case ROCKSDB_NAMESPACE::Histograms::FILE_READ_COMPACTION_MICROS: return 0x3B; + case ROCKSDB_NAMESPACE::Histograms::FILE_READ_DB_OPEN_MICROS: + return 0x3C; case ROCKSDB_NAMESPACE::Histograms::HISTOGRAM_ENUM_MAX: // 0x1F for backwards compatibility on current minor version. return 0x1F; @@ -5750,6 +5752,8 @@ class HistogramTypeJni { return ROCKSDB_NAMESPACE::Histograms::FILE_READ_FLUSH_MICROS; case 0x3B: return ROCKSDB_NAMESPACE::Histograms::FILE_READ_COMPACTION_MICROS; + case 0x3C: + return ROCKSDB_NAMESPACE::Histograms::FILE_READ_DB_OPEN_MICROS; case 0x1F: // 0x1F for backwards compatibility on current minor version. return ROCKSDB_NAMESPACE::Histograms::HISTOGRAM_ENUM_MAX; diff --git a/java/src/main/java/org/rocksdb/HistogramType.java b/java/src/main/java/org/rocksdb/HistogramType.java index 35724a108..aad0d9550 100644 --- a/java/src/main/java/org/rocksdb/HistogramType.java +++ b/java/src/main/java/org/rocksdb/HistogramType.java @@ -173,6 +173,8 @@ public enum HistogramType { FILE_READ_COMPACTION_MICROS((byte) 0x3B), + FILE_READ_DB_OPEN_MICROS((byte) 0x3C), + // 0x1F for backwards compatibility on current minor version. HISTOGRAM_ENUM_MAX((byte) 0x1F); diff --git a/monitoring/statistics.cc b/monitoring/statistics.cc index 7459dc46f..65bc57544 100644 --- a/monitoring/statistics.cc +++ b/monitoring/statistics.cc @@ -256,6 +256,7 @@ const std::vector> HistogramsNameMap = { {SST_READ_MICROS, "rocksdb.sst.read.micros"}, {FILE_READ_FLUSH_MICROS, "rocksdb.file.read.flush.micros"}, {FILE_READ_COMPACTION_MICROS, "rocksdb.file.read.compaction.micros"}, + {FILE_READ_DB_OPEN_MICROS, "rocksdb.file.read.db.open.micros"}, {NUM_SUBCOMPACTIONS_SCHEDULED, "rocksdb.num.subcompactions.scheduled"}, {BYTES_PER_READ, "rocksdb.bytes.per.read"}, {BYTES_PER_WRITE, "rocksdb.bytes.per.write"},