From 17a6f7faaff4461899e571589d199d542c7f9ba7 Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Thu, 30 Jun 2022 11:03:52 -0700 Subject: [PATCH] Add load_latest_options() to C api (#10152) Summary: Add load_latest_options() to C api. Pull Request resolved: https://github.com/facebook/rocksdb/pull/10152 Test Plan: Extend the existing c_test by reopening db using the latest options file at different parts of the test. Reviewed By: akankshamahajan15 Differential Revision: D37305225 Pulled By: ajkr fbshipit-source-id: 8b3bab73f56fa6fcbdba45aae393145d007b3962 --- db/c.cc | 51 ++++++++++++++++++++++++ db/c_test.c | 97 ++++++++++++++++++++++++++++++++++++++++++++- include/rocksdb/c.h | 22 ++++++++++ 3 files changed, 168 insertions(+), 2 deletions(-) diff --git a/db/c.cc b/db/c.cc index bc6f8efc9..a820159c3 100644 --- a/db/c.cc +++ b/db/c.cc @@ -41,6 +41,7 @@ #include "rocksdb/utilities/db_ttl.h" #include "rocksdb/utilities/memory_util.h" #include "rocksdb/utilities/optimistic_transaction_db.h" +#include "rocksdb/utilities/options_util.h" #include "rocksdb/utilities/table_properties_collectors.h" #include "rocksdb/utilities/transaction.h" #include "rocksdb/utilities/transaction_db.h" @@ -2544,6 +2545,56 @@ void rocksdb_write_writebatch_wi( SaveError(errptr, db->rep->Write(options->rep, wb)); } +void rocksdb_load_latest_options( + const char* db_path, rocksdb_env_t* env, bool ignore_unknown_options, + rocksdb_cache_t* cache, rocksdb_options_t** db_options, + size_t* num_column_families, char*** list_column_family_names, + rocksdb_options_t*** list_column_family_options, char** errptr) { + DBOptions db_opt; + std::vector cf_descs; + Status s = LoadLatestOptions(std::string(db_path), env->rep, &db_opt, + &cf_descs, ignore_unknown_options, &cache->rep); + if (s.ok()) { + char** cf_names = (char**)malloc(cf_descs.size() * sizeof(char*)); + rocksdb_options_t** cf_options = (rocksdb_options_t**)malloc( + cf_descs.size() * sizeof(rocksdb_options_t*)); + for (size_t i = 0; i < cf_descs.size(); ++i) { + cf_names[i] = strdup(cf_descs[i].name.c_str()); + cf_options[i] = new rocksdb_options_t{ + Options(DBOptions(), std::move(cf_descs[i].options))}; + } + *num_column_families = cf_descs.size(); + *db_options = new rocksdb_options_t{ + Options(std::move(db_opt), ColumnFamilyOptions())}; + *list_column_family_names = cf_names; + *list_column_family_options = cf_options; + } else { + *num_column_families = 0; + *db_options = nullptr; + *list_column_family_names = nullptr; + *list_column_family_options = nullptr; + SaveError(errptr, s); + } +} + +void rocksdb_load_latest_options_destroy( + rocksdb_options_t* db_options, char** list_column_family_names, + rocksdb_options_t** list_column_family_options, size_t len) { + rocksdb_options_destroy(db_options); + if (list_column_family_names) { + for (size_t i = 0; i < len; ++i) { + free(list_column_family_names[i]); + } + free(list_column_family_names); + } + if (list_column_family_options) { + for (size_t i = 0; i < len; ++i) { + rocksdb_options_destroy(list_column_family_options[i]); + } + free(list_column_family_options); + } +} + rocksdb_block_based_table_options_t* rocksdb_block_based_options_create() { return new rocksdb_block_based_table_options_t; diff --git a/db/c_test.c b/db/c_test.c index c988dd299..e28c597f5 100644 --- a/db/c_test.c +++ b/db/c_test.c @@ -561,6 +561,71 @@ static void CheckTxnDBPinGetCF(rocksdb_transactiondb_t* txn_db, rocksdb_pinnableslice_destroy(p); } +static void LoadAndCheckLatestOptions(const char* db_name, rocksdb_env_t* env, + bool ignore_unknown_options, + rocksdb_cache_t* cache, + rocksdb_comparator_t* cmp, + const size_t expected_num_column_families, + const char** expected_cf_names, + const char* expected_open_err) { + rocksdb_options_t* db_options; + size_t num_column_families; + char** list_column_family_names; + rocksdb_options_t** list_column_family_options; + char* err = 0; + + // load the latest rocksdb option + rocksdb_load_latest_options(db_name, env, ignore_unknown_options, cache, + &db_options, &num_column_families, + &list_column_family_names, + &list_column_family_options, &err); + assert(num_column_families == expected_num_column_families); + CheckNoError(err); + + // verify the loaded options by opening the db. + rocksdb_options_set_error_if_exists(db_options, 0); + + char** list_const_cf_names = + (char**)malloc(num_column_families * sizeof(char*)); + rocksdb_options_t** list_const_cf_options = (rocksdb_options_t**)malloc( + num_column_families * sizeof(rocksdb_options_t*)); + for (size_t i = 0; i < num_column_families; ++i) { + assert(strcmp(list_column_family_names[i], expected_cf_names[i]) == 0); + list_const_cf_names[i] = list_column_family_names[i]; + if (cmp) { + rocksdb_options_set_comparator(list_column_family_options[i], cmp); + } + list_const_cf_options[i] = list_column_family_options[i]; + } + rocksdb_column_family_handle_t** handles = + (rocksdb_column_family_handle_t**)malloc( + num_column_families * sizeof(rocksdb_column_family_handle_t*)); + + rocksdb_t* db = rocksdb_open_column_families( + db_options, db_name, (int)num_column_families, + (const char* const*)list_const_cf_names, + (const rocksdb_options_t* const*)list_const_cf_options, handles, &err); + if (expected_open_err == NULL) { + CheckNoError(err); + for (size_t i = 0; i < num_column_families; ++i) { + rocksdb_column_family_handle_destroy(handles[i]); + } + free(handles); + rocksdb_close(db); + } else { + assert(err != NULL); + assert(strcmp(err, expected_open_err) == 0); + free(handles); + free(err); + } + + free(list_const_cf_names); + free(list_const_cf_options); + rocksdb_load_latest_options_destroy(db_options, list_column_family_names, + list_column_family_options, + num_column_families); +} + int main(int argc, char** argv) { (void)argc; (void)argv; @@ -1341,7 +1406,6 @@ int main(int argc, char** argv) { rocksdb_merge(db, woptions, "bar", 3, "barvalue", 8, &err); CheckNoError(err); CheckGet(db, roptions, "bar", "fake"); - } StartPhase("columnfamilies"); @@ -1353,7 +1417,16 @@ int main(int argc, char** argv) { rocksdb_options_t* db_options = rocksdb_options_create(); rocksdb_options_set_create_if_missing(db_options, 1); db = rocksdb_open(db_options, dbname, &err); - CheckNoError(err) + CheckNoError(err); + rocksdb_close(db); + { + const char* expected_cf_names[1] = {"default"}; + LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 1, + expected_cf_names, NULL); + } + + rocksdb_options_set_create_if_missing(db_options, 0); + db = rocksdb_open(db_options, dbname, &err); rocksdb_column_family_handle_t* cfh; cfh = rocksdb_create_column_family(db, db_options, "cf1", &err); rocksdb_column_family_handle_destroy(cfh); @@ -1373,6 +1446,10 @@ int main(int argc, char** argv) { const char* cf_names[2] = {"default", "cf1"}; const rocksdb_options_t* cf_opts[2] = {cf_options, cf_options}; rocksdb_column_family_handle_t* handles[2]; + + LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 2, cf_names, + NULL); + db = rocksdb_open_column_families(db_options, dbname, 2, cf_names, cf_opts, handles, &err); CheckNoError(err); @@ -1545,6 +1622,12 @@ int main(int argc, char** argv) { rocksdb_column_family_handle_destroy(handles[i]); } rocksdb_close(db); + { + // As column family has been dropped, we expect only one column family. + const char* expected_cf_names[1] = {"default"}; + LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 1, + expected_cf_names, NULL); + } rocksdb_destroy_db(options, dbname, &err); rocksdb_options_destroy(db_options); rocksdb_options_destroy(cf_options); @@ -1606,6 +1689,16 @@ int main(int argc, char** argv) { rocksdb_readoptions_set_total_order_seek(roptions, 0); rocksdb_close(db); + + { + const char* expected_cf_names[1] = {"default"}; + LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 1, + expected_cf_names, + "Invalid argument: leveldb.BytewiseComparator: " + "does not match existing comparator foo"); + LoadAndCheckLatestOptions(dbname, env, false, cache, cmp, 1, + expected_cf_names, NULL); + } rocksdb_destroy_db(options, dbname, &err); } diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index 65fc57136..a1ce50c5c 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -949,6 +949,28 @@ extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* rocksdb_writebatch_wi_create_iter rocksdb_iterator_t* base_iterator, rocksdb_column_family_handle_t* cf); +/* Options utils */ + +// Load the latest rocksdb options from the specified db_path. +// +// On success, num_column_families will be updated with a non-zero +// number indicating the number of column families. +// The returned db_options, column_family_names, and column_family_options +// should be released via rocksdb_load_latest_options_destroy(). +// +// On error, a non-null errptr that includes the error message will be +// returned. db_options, column_family_names, and column_family_options +// will be set to NULL. +extern ROCKSDB_LIBRARY_API void rocksdb_load_latest_options( + const char* db_path, rocksdb_env_t* env, bool ignore_unknown_options, + rocksdb_cache_t* cache, rocksdb_options_t** db_options, + size_t* num_column_families, char*** column_family_names, + rocksdb_options_t*** column_family_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_load_latest_options_destroy( + rocksdb_options_t* db_options, char** list_column_family_names, + rocksdb_options_t** list_column_family_options, size_t len); + /* Block based table options */ extern ROCKSDB_LIBRARY_API rocksdb_block_based_table_options_t*