NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Memory usage stats in C API (#4340)

Browse Source 
Summary:
Please consider this small PR providing access to the `MemoryUsage::GetApproximateMemoryUsageByType` function in plain C API. Actually I'm working on Go application and now trying to investigate the reasons of high memory consumption (#4313). Go [wrappers](https://github.com/tecbot/gorocksdb) are built on the top of Rocksdb C API. According to the #706, `MemoryUsage::GetApproximateMemoryUsageByType` is considered as the best option to get database internal memory usage stats, but it wasn't supported in C API yet.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4340

Differential Revision: D9655135

Pulled By: ajkr

fbshipit-source-id: a3d2f3f47c143ae75862fbcca2f571ea1b49e14a
main
Vitaly Isaev 6 years ago committed by Facebook Github Bot
parent 9ea9007b50
commit 0bd2ede10e
3 changed files with 172 additions and 1 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 101
      db/c.cc
  2. 43
      db/c_test.c
  3. 29
      include/rocksdb/c.h

101
db/c.cc
Unescape View File

@ -33,6 +33,7 @@
#include "rocksdb/utilities/backupable_db.h"
#include "rocksdb/utilities/checkpoint.h"
#include "rocksdb/utilities/db_ttl.h"
#include "rocksdb/utilities/memory_util.h"
#include "rocksdb/utilities/optimistic_transaction_db.h"
#include "rocksdb/utilities/transaction.h"
#include "rocksdb/utilities/transaction_db.h"
@ -41,6 +42,10 @@
#include "rocksdb/perf_context.h"
#include "utilities/merge_operators.h"
#include <vector>
#include <unordered_set>
#include <map>
using rocksdb::BytewiseComparator;
using rocksdb::Cache;
using rocksdb::ColumnFamilyDescriptor;
@ -108,8 +113,12 @@ using rocksdb::TransactionLogIterator;
using rocksdb::BatchResult;
using rocksdb::PerfLevel;
using rocksdb::PerfContext;
using rocksdb::MemoryUtil;
using std::shared_ptr;
using std::vector;
using std::unordered_set;
using std::map;
extern "C" {
@ -4101,6 +4110,98 @@ const char* rocksdb_pinnableslice_value(const rocksdb_pinnableslice_t* v,
*vlen = v->rep.size();
return v->rep.data();
}
// container to keep databases and caches in order to use rocksdb::MemoryUtil
struct rocksdb_memory_consumers_t {
std::vector<rocksdb_t*> dbs;
std::unordered_set<rocksdb_cache_t*> caches;
};
// initializes new container of memory consumers
rocksdb_memory_consumers_t* rocksdb_memory_consumers_create() {
return new rocksdb_memory_consumers_t;
}
// adds datatabase to the container of memory consumers
void rocksdb_memory_consumers_add_db(rocksdb_memory_consumers_t* consumers,
rocksdb_t* db) {
consumers->dbs.push_back(db);
}
// adds cache to the container of memory consumers
void rocksdb_memory_consumers_add_cache(rocksdb_memory_consumers_t* consumers,
rocksdb_cache_t* cache) {
consumers->caches.insert(cache);
}
// deletes container with memory consumers
void rocksdb_memory_consumers_destroy(rocksdb_memory_consumers_t* consumers) {
delete consumers;
}
// contains memory usage statistics provided by rocksdb::MemoryUtil
struct rocksdb_memory_usage_t {
uint64_t mem_table_total;
uint64_t mem_table_unflushed;
uint64_t mem_table_readers_total;
uint64_t cache_total;
};
// estimates amount of memory occupied by consumers (dbs and caches)
rocksdb_memory_usage_t* rocksdb_approximate_memory_usage_create(
rocksdb_memory_consumers_t* consumers, char** errptr) {
vector<DB*> dbs;
for (auto db : consumers->dbs) {
dbs.push_back(db->rep);
}
unordered_set<const Cache*> cache_set;
for (auto cache : consumers->caches) {
cache_set.insert(const_cast<const Cache*>(cache->rep.get()));
}
std::map<rocksdb::MemoryUtil::UsageType, uint64_t> usage_by_type;
auto status = MemoryUtil::GetApproximateMemoryUsageByType(dbs, cache_set,
&usage_by_type);
if (SaveError(errptr, status)) {
return nullptr;
}
auto result = new rocksdb_memory_usage_t;
result->mem_table_total = usage_by_type[MemoryUtil::kMemTableTotal];
result->mem_table_unflushed = usage_by_type[MemoryUtil::kMemTableUnFlushed];
result->mem_table_readers_total = usage_by_type[MemoryUtil::kTableReadersTotal];
result->cache_total = usage_by_type[MemoryUtil::kCacheTotal];
return result;
}
uint64_t rocksdb_approximate_memory_usage_get_mem_table_total(
rocksdb_memory_usage_t* memory_usage) {
return memory_usage->mem_table_total;
}
uint64_t rocksdb_approximate_memory_usage_get_mem_table_unflushed(
rocksdb_memory_usage_t* memory_usage) {
return memory_usage->mem_table_unflushed;
}
uint64_t rocksdb_approximate_memory_usage_get_mem_table_readers_total(
rocksdb_memory_usage_t* memory_usage) {
return memory_usage->mem_table_readers_total;
}
uint64_t rocksdb_approximate_memory_usage_get_cache_total(
rocksdb_memory_usage_t* memory_usage) {
return memory_usage->cache_total;
}
// deletes container with memory usage estimates
void rocksdb_approximate_memory_usage_destroy(rocksdb_memory_usage_t* usage) {
delete usage;
}
} // end extern "C"
#endif // !ROCKSDB_LITE

43
db/c_test.c
Unescape View File

@ -1334,6 +1334,47 @@ int main(int argc, char** argv) {
rocksdb_destroy_db(options, dbname, &err);
}
// Check memory usage stats
StartPhase("approximate_memory_usage");
{
// Create database
db = rocksdb_open(options, dbname, &err);
CheckNoError(err);
rocksdb_memory_consumers_t* consumers;
consumers = rocksdb_memory_consumers_create();
rocksdb_memory_consumers_add_db(consumers, db);
rocksdb_memory_consumers_add_cache(consumers, cache);
// take memory usage report before write-read operation
rocksdb_memory_usage_t* mu1;
mu1 = rocksdb_approximate_memory_usage_create(consumers, &err);
CheckNoError(err);
// Put data (this should affect memtables)
rocksdb_put(db, woptions, "memory", 6, "test", 4, &err);
CheckNoError(err);
CheckGet(db, roptions, "memory", "test");
// take memory usage report after write-read operation
rocksdb_memory_usage_t* mu2;
mu2 = rocksdb_approximate_memory_usage_create(consumers, &err);
CheckNoError(err);
// amount of memory used within memtables should grow
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_total(mu2) >=
rocksdb_approximate_memory_usage_get_mem_table_total(mu1));
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu2) >=
rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu1));
rocksdb_memory_consumers_destroy(consumers);
rocksdb_approximate_memory_usage_destroy(mu1);
rocksdb_approximate_memory_usage_destroy(mu2);
rocksdb_close(db);
rocksdb_destroy_db(options, dbname, &err);
CheckNoError(err);
}
StartPhase("cuckoo_options");
{
rocksdb_cuckoo_table_options_t* cuckoo_options;
@ -1675,7 +1716,7 @@ int main(int argc, char** argv) {
db = rocksdb_open(options, dbname, &err);
CheckNoError(err);
}
StartPhase("cleanup");
rocksdb_close(db);
rocksdb_options_destroy(options);

29
include/rocksdb/c.h
Unescape View File

@ -123,6 +123,8 @@ typedef struct rocksdb_transaction_t rocksdb_transaction_t;
typedef struct rocksdb_checkpoint_t rocksdb_checkpoint_t;
typedef struct rocksdb_wal_iterator_t rocksdb_wal_iterator_t;
typedef struct rocksdb_wal_readoptions_t rocksdb_wal_readoptions_t;
typedef struct rocksdb_memory_consumers_t rocksdb_memory_consumers_t;
typedef struct rocksdb_memory_usage_t rocksdb_memory_usage_t;
/* DB operations */
@ -1672,6 +1674,33 @@ extern ROCKSDB_LIBRARY_API void rocksdb_pinnableslice_destroy(
extern ROCKSDB_LIBRARY_API const char* rocksdb_pinnableslice_value(
const rocksdb_pinnableslice_t* t, size_t* vlen);
extern ROCKSDB_LIBRARY_API rocksdb_memory_consumers_t*
rocksdb_memory_consumers_create();
extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_db(
rocksdb_memory_consumers_t* consumers, rocksdb_t* db);
extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_cache(
rocksdb_memory_consumers_t* consumers, rocksdb_cache_t* cache);
extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_destroy(
rocksdb_memory_consumers_t* consumers);
extern ROCKSDB_LIBRARY_API rocksdb_memory_usage_t*
rocksdb_approximate_memory_usage_create(rocksdb_memory_consumers_t* consumers,
char** errptr);
extern ROCKSDB_LIBRARY_API void rocksdb_approximate_memory_usage_destroy(
rocksdb_memory_usage_t* usage);
extern ROCKSDB_LIBRARY_API uint64_t
rocksdb_approximate_memory_usage_get_mem_table_total(
rocksdb_memory_usage_t* memory_usage);
extern ROCKSDB_LIBRARY_API uint64_t
rocksdb_approximate_memory_usage_get_mem_table_unflushed(
rocksdb_memory_usage_t* memory_usage);
extern ROCKSDB_LIBRARY_API uint64_t
rocksdb_approximate_memory_usage_get_mem_table_readers_total(
rocksdb_memory_usage_t* memory_usage);
extern ROCKSDB_LIBRARY_API uint64_t
rocksdb_approximate_memory_usage_get_cache_total(
rocksdb_memory_usage_t* memory_usage);
#ifdef __cplusplus
} /* end extern "C" */
#endif

Write Preview
Loading…
Cancel
Save