diff --git a/HISTORY.md b/HISTORY.md index 339aa4c2e..fd5b56748 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -15,6 +15,7 @@ ### New Features * User defined timestamp feature supports `CompactRange` and `GetApproximateSizes`. +* Support getting aggregated table properties (kAggregatedTableProperties and kAggregatedTablePropertiesAtLevel) with DB::GetMapProperty, for easier access to the data in a structured format. * Experimental option BlockBasedTableOptions::optimize_filters_for_memory now works with experimental Ribbon filter (as well as Bloom filter). ### Public API Change diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 7537abdd2..512bc1b01 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -378,10 +378,11 @@ const std::unordered_map {false, &InternalStats::HandleSsTables, nullptr, nullptr, nullptr}}, {DB::Properties::kAggregatedTableProperties, {false, &InternalStats::HandleAggregatedTableProperties, nullptr, - nullptr, nullptr}}, + &InternalStats::HandleAggregatedTablePropertiesMap, nullptr}}, {DB::Properties::kAggregatedTablePropertiesAtLevel, {false, &InternalStats::HandleAggregatedTablePropertiesAtLevel, - nullptr, nullptr, nullptr}}, + nullptr, &InternalStats::HandleAggregatedTablePropertiesAtLevelMap, + nullptr}}, {DB::Properties::kNumImmutableMemTable, {false, nullptr, &InternalStats::HandleNumImmutableMemTable, nullptr, nullptr}}, @@ -510,11 +511,12 @@ bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info, } bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info, - const Slice& /*property*/, + const Slice& property, std::map* value) { assert(value != nullptr); assert(property_info.handle_map != nullptr); - return (this->*(property_info.handle_map))(value); + Slice arg = GetPropertyNameAndArg(property).second; + return (this->*(property_info.handle_map))(value, arg); } bool InternalStats::GetIntProperty(const DBPropertyInfo& property_info, @@ -590,7 +592,7 @@ bool InternalStats::HandleStats(std::string* value, Slice suffix) { } bool InternalStats::HandleCFMapStats( - std::map* cf_stats) { + std::map* cf_stats, Slice /*suffix*/) { DumpCFMapStats(cf_stats); return true; } @@ -634,7 +636,27 @@ bool InternalStats::HandleAggregatedTableProperties(std::string* value, return true; } -bool InternalStats::HandleAggregatedTablePropertiesAtLevel(std::string* value, +static std::map MapUint64ValuesToString( + const std::map& from) { + std::map to; + for (const auto& e : from) { + to[e.first] = ToString(e.second); + } + return to; +} + +bool InternalStats::HandleAggregatedTablePropertiesMap( + std::map* values, Slice /*suffix*/) { + std::shared_ptr tp; + auto s = cfd_->current()->GetAggregatedTableProperties(&tp); + if (!s.ok()) { + return false; + } + *values = MapUint64ValuesToString(tp->GetAggregatablePropertiesAsMap()); + return true; +} + +bool InternalStats::HandleAggregatedTablePropertiesAtLevel(std::string* values, Slice suffix) { uint64_t level; bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty(); @@ -647,7 +669,24 @@ bool InternalStats::HandleAggregatedTablePropertiesAtLevel(std::string* value, if (!s.ok()) { return false; } - *value = tp->ToString(); + *values = tp->ToString(); + return true; +} + +bool InternalStats::HandleAggregatedTablePropertiesAtLevelMap( + std::map* values, Slice suffix) { + uint64_t level; + bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty(); + if (!ok || static_cast(level) >= number_levels_) { + return false; + } + std::shared_ptr tp; + auto s = cfd_->current()->GetAggregatedTableProperties( + &tp, static_cast(level)); + if (!s.ok()) { + return false; + } + *values = MapUint64ValuesToString(tp->GetAggregatablePropertiesAsMap()); return true; } diff --git a/db/internal_stats.h b/db/internal_stats.h index 61d9b7657..056719c5c 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -44,7 +44,9 @@ struct DBPropertyInfo { Version* version); // @param props Map of general properties to populate - bool (InternalStats::*handle_map)(std::map* props); + // @param suffix Argument portion of the property. (see handle_string) + bool (InternalStats::*handle_map)(std::map* props, + Slice suffix); // handle the string type properties rely on DBImpl methods // @param value Value-result argument for storing the property's string value @@ -525,7 +527,8 @@ class InternalStats { bool HandleCompressionRatioAtLevelPrefix(std::string* value, Slice suffix); bool HandleLevelStats(std::string* value, Slice suffix); bool HandleStats(std::string* value, Slice suffix); - bool HandleCFMapStats(std::map* compaction_stats); + bool HandleCFMapStats(std::map* compaction_stats, + Slice suffix); bool HandleCFStats(std::string* value, Slice suffix); bool HandleCFStatsNoFileHistogram(std::string* value, Slice suffix); bool HandleCFFileHistogram(std::string* value, Slice suffix); @@ -533,6 +536,10 @@ class InternalStats { bool HandleSsTables(std::string* value, Slice suffix); bool HandleAggregatedTableProperties(std::string* value, Slice suffix); bool HandleAggregatedTablePropertiesAtLevel(std::string* value, Slice suffix); + bool HandleAggregatedTablePropertiesMap( + std::map* values, Slice suffix); + bool HandleAggregatedTablePropertiesAtLevelMap( + std::map* values, Slice suffix); bool HandleNumImmutableMemTable(uint64_t* value, DBImpl* db, Version* version); bool HandleNumImmutableMemTableFlushed(uint64_t* value, DBImpl* db, diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 6ef02121f..995d9f0f1 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -713,7 +713,9 @@ class DB { virtual void ReleaseSnapshot(const Snapshot* snapshot) = 0; #ifndef ROCKSDB_LITE - // Contains all valid property arguments for GetProperty(). + // Contains all valid property arguments for GetProperty() or + // GetMapProperty(). Each is a "string" property for retrieval with + // GetProperty() unless noted as a "map" property, for GetMapProperty(). // // NOTE: Property names cannot end in numbers since those are interpreted as // arguments, e.g., see kNumFilesAtLevelPrefix. @@ -738,19 +740,14 @@ class DB { // SST files. static const std::string kSSTables; - // "rocksdb.cfstats" - Both of "rocksdb.cfstats-no-file-histogram" and - // "rocksdb.cf-file-histogram" together. See below for description - // of the two. + // "rocksdb.cfstats" - Raw data from "rocksdb.cfstats-no-file-histogram" + // and "rocksdb.cf-file-histogram" as a "map" property. static const std::string kCFStats; // "rocksdb.cfstats-no-file-histogram" - returns a multi-line string with // general columm family stats per-level over db's lifetime ("L"), // aggregated over db's lifetime ("Sum"), and aggregated over the // interval since the last retrieval ("Int"). - // It could also be used to return the stats in the format of the map. - // In this case there will a pair of string to array of double for - // each level as well as for "Sum". "Int" stats will not be affected - // when this form of stats are retrieved. static const std::string kCFStatsNoFileHistogram; // "rocksdb.cf-file-histogram" - print out how many file reads to every @@ -891,8 +888,10 @@ class DB { // based. static const std::string kEstimatePendingCompactionBytes; - // "rocksdb.aggregated-table-properties" - returns a string representation - // of the aggregated table properties of the target column family. + // "rocksdb.aggregated-table-properties" - returns a string or map + // representation of the aggregated table properties of the target + // column family. Only properties that make sense for aggregation + // are included. static const std::string kAggregatedTableProperties; // "rocksdb.aggregated-table-properties-at-level", same as the previous @@ -930,15 +929,19 @@ class DB { }; #endif /* ROCKSDB_LITE */ - // DB implementations can export properties about their state via this method. - // If "property" is a valid property understood by this DB implementation (see - // Properties struct above for valid options), fills "*value" with its current - // value and returns true. Otherwise, returns false. + // DB implementations export properties about their state via this method. + // If "property" is a valid "string" property understood by this DB + // implementation (see Properties struct above for valid options), fills + // "*value" with its current value and returns true. Otherwise, returns + // false. virtual bool GetProperty(ColumnFamilyHandle* column_family, const Slice& property, std::string* value) = 0; virtual bool GetProperty(const Slice& property, std::string* value) { return GetProperty(DefaultColumnFamily(), property, value); } + + // Like GetProperty but for valid "map" properties. (Some properties can be + // accessed as either "string" properties or "map" properties.) virtual bool GetMapProperty(ColumnFamilyHandle* column_family, const Slice& property, std::map* value) = 0; diff --git a/include/rocksdb/table_properties.h b/include/rocksdb/table_properties.h index c6810aa55..9f2b64866 100644 --- a/include/rocksdb/table_properties.h +++ b/include/rocksdb/table_properties.h @@ -258,6 +258,11 @@ struct TableProperties { // Aggregate the numerical member variables of the specified // TableProperties. void Add(const TableProperties& tp); + + // Subset of properties that make sense when added together + // between tables. Keys match field names in this class instead + // of using full property names. + std::map GetAggregatablePropertiesAsMap() const; }; // Extra properties diff --git a/table/table_properties.cc b/table/table_properties.cc index 6af1f379f..310fb4a0e 100644 --- a/table/table_properties.cc +++ b/table/table_properties.cc @@ -193,6 +193,24 @@ void TableProperties::Add(const TableProperties& tp) { num_range_deletions += tp.num_range_deletions; } +std::map +TableProperties::GetAggregatablePropertiesAsMap() const { + std::map rv; + rv["data_size"] = data_size; + rv["index_size"] = index_size; + rv["index_partitions"] = index_partitions; + rv["top_level_index_size"] = top_level_index_size; + rv["filter_size"] = filter_size; + rv["raw_key_size"] = raw_key_size; + rv["raw_value_size"] = raw_value_size; + rv["num_data_blocks"] = num_data_blocks; + rv["num_entries"] = num_entries; + rv["num_deletions"] = num_deletions; + rv["num_merge_operands"] = num_merge_operands; + rv["num_range_deletions"] = num_range_deletions; + return rv; +} + const std::string TablePropertiesNames::kDbId = "rocksdb.creating.db.identity"; const std::string TablePropertiesNames::kDbSessionId = "rocksdb.creating.session.identity"; diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index 8ff980025..75aca8076 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -227,6 +227,10 @@ LDBCommand* LDBCommand::SelectCommand(const ParsedParams& parsed_params) { return new FileChecksumDumpCommand(parsed_params.cmd_params, parsed_params.option_map, parsed_params.flags); + } else if (parsed_params.cmd == GetPropertyCommand::Name()) { + return new GetPropertyCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); } else if (parsed_params.cmd == ListColumnFamiliesCommand::Name()) { return new ListColumnFamiliesCommand(parsed_params.cmd_params, parsed_params.option_map, @@ -1259,6 +1263,58 @@ void FileChecksumDumpCommand::DoCommand() { // ---------------------------------------------------------------------------- +void GetPropertyCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(GetPropertyCommand::Name()); + ret.append(" "); + ret.append("\n"); +} + +GetPropertyCommand::GetPropertyCommand( + const std::vector& params, + const std::map& options, + const std::vector& flags) + : LDBCommand(options, flags, true, BuildCmdLineOptions({})) { + if (params.size() != 1) { + exec_state_ = + LDBCommandExecuteResult::Failed("property name must be specified"); + } else { + property_ = params[0]; + } +} + +void GetPropertyCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + + std::map value_map; + std::string value; + + // Rather than having different ldb command for map properties vs. string + // properties, we simply try Map property first. (This order only chosen + // because I prefer the map-style output for + // "rocksdb.aggregated-table-properties".) + if (db_->GetMapProperty(GetCfHandle(), property_, &value_map)) { + if (value_map.empty()) { + fprintf(stdout, "%s: \n", property_.c_str()); + } else { + for (auto& e : value_map) { + fprintf(stdout, "%s.%s: %s\n", property_.c_str(), e.first.c_str(), + e.second.c_str()); + } + } + } else if (db_->GetProperty(GetCfHandle(), property_, &value)) { + fprintf(stdout, "%s: %s\n", property_.c_str(), value.c_str()); + } else { + exec_state_ = + LDBCommandExecuteResult::Failed("failed to get property: " + property_); + } +} + +// ---------------------------------------------------------------------------- + void ListColumnFamiliesCommand::Help(std::string& ret) { ret.append(" "); ret.append(ListColumnFamiliesCommand::Name()); diff --git a/tools/ldb_cmd_impl.h b/tools/ldb_cmd_impl.h index 475cc44c9..e21251f9d 100644 --- a/tools/ldb_cmd_impl.h +++ b/tools/ldb_cmd_impl.h @@ -190,6 +190,21 @@ class FileChecksumDumpCommand : public LDBCommand { static const std::string ARG_PATH; }; +class GetPropertyCommand : public LDBCommand { + public: + static std::string Name() { return "get_property"; } + + GetPropertyCommand(const std::vector& params, + const std::map& options, + const std::vector& flags); + + static void Help(std::string& ret); + void DoCommand() override; + + private: + std::string property_; +}; + class ListColumnFamiliesCommand : public LDBCommand { public: static std::string Name() { return "list_column_families"; } diff --git a/tools/ldb_test.py b/tools/ldb_test.py index 46edfa4e5..6587ad6c3 100644 --- a/tools/ldb_test.py +++ b/tools/ldb_test.py @@ -473,6 +473,27 @@ class LDBTestCase(unittest.TestCase): expected_pattern, unexpected=False, isPattern=True) + def testGetProperty(self): + print("Running testGetProperty...") + dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) + self.assertRunOK("put 1 1 --create_if_missing", "OK") + self.assertRunOK("put 2 2", "OK") + # A "string" property + cmd = "--db=%s get_property rocksdb.estimate-num-keys" + self.assertRunOKFull(cmd % dbPath, + "rocksdb.estimate-num-keys: 2") + # A "map" property + # FIXME: why doesn't this pick up two entries? + cmd = "--db=%s get_property rocksdb.aggregated-table-properties" + part = "rocksdb.aggregated-table-properties.num_entries: " + expected_pattern = re.compile(part) + self.assertRunOKFull(cmd % dbPath, + expected_pattern, unexpected=False, + isPattern=True) + # An invalid property + cmd = "--db=%s get_property rocksdb.this-property-does-not-exist" + self.assertRunFAILFull(cmd % dbPath) + def testSSTDump(self): print("Running testSSTDump...") diff --git a/tools/ldb_tool.cc b/tools/ldb_tool.cc index 696737bdc..956244792 100644 --- a/tools/ldb_tool.cc +++ b/tools/ldb_tool.cc @@ -87,6 +87,7 @@ void LDBCommandRunner::PrintHelp(const LDBOptions& ldb_options, DBLoaderCommand::Help(ret); ManifestDumpCommand::Help(ret); FileChecksumDumpCommand::Help(ret); + GetPropertyCommand::Help(ret); ListColumnFamiliesCommand::Help(ret); CreateColumnFamilyCommand::Help(ret); DropColumnFamilyCommand::Help(ret);