From 86e2a1eeea07917923318c5d7ed95b6aafe652bb Mon Sep 17 00:00:00 2001 From: Marko Kevac Date: Sat, 31 Jan 2015 15:47:49 +0300 Subject: [PATCH 1/5] Allow creating backups from C --- db/c.cc | 30 ++++++++++++++++++++++++++++++ examples/c_simple_example.c | 9 +++++++++ include/rocksdb/c.h | 13 +++++++++++++ 3 files changed, 52 insertions(+) diff --git a/db/c.cc b/db/c.cc index 76a949cd1..c952b2e8b 100644 --- a/db/c.cc +++ b/db/c.cc @@ -29,6 +29,7 @@ #include "rocksdb/statistics.h" #include "rocksdb/slice_transform.h" #include "rocksdb/table.h" +#include "rocksdb/utilities/backupable_db.h" using rocksdb::Cache; using rocksdb::ColumnFamilyDescriptor; @@ -69,12 +70,15 @@ using rocksdb::WritableFile; using rocksdb::WriteBatch; using rocksdb::WriteOptions; using rocksdb::LiveFileMetaData; +using rocksdb::BackupEngine; +using rocksdb::BackupableDBOptions; using std::shared_ptr; extern "C" { struct rocksdb_t { DB* rep; }; +struct rocksdb_backup_engine_t { BackupEngine* rep; }; struct rocksdb_iterator_t { Iterator* rep; }; struct rocksdb_writebatch_t { WriteBatch rep; }; struct rocksdb_snapshot_t { const Snapshot* rep; }; @@ -527,6 +531,32 @@ rocksdb_t* rocksdb_open_for_read_only( return result; } +rocksdb_backup_engine_t* rocksdb_backup_engine_open( + const char* path, + char** errptr) { + BackupEngine* be; + if (SaveError(errptr, BackupEngine::Open(Env::Default(), BackupableDBOptions(path), &be))) { + return nullptr; + } + rocksdb_backup_engine_t* result = new rocksdb_backup_engine_t; + result->rep = be; + return result; +} + +void rocksdb_backup_engine_create_new_backup( + rocksdb_backup_engine_t *be, + rocksdb_t *db, + char** errptr) { + SaveError(errptr, be->rep->CreateNewBackup(db->rep)); +} + +void rocksdb_backup_engine_close( + rocksdb_backup_engine_t *be) { + delete be->rep; + delete be; +} + + void rocksdb_close(rocksdb_t* db) { delete db->rep; delete db; diff --git a/examples/c_simple_example.c b/examples/c_simple_example.c index 1dd380721..8340026db 100644 --- a/examples/c_simple_example.c +++ b/examples/c_simple_example.c @@ -8,9 +8,11 @@ #include // sysconf() - get CPU count const char DBPath[] = "/tmp/rocksdb_simple_example"; +const char DBBackupPath[] = "/tmp/rocksdb_simple_example_backup"; int main(int argc, char **argv) { rocksdb_t *db; + rocksdb_backup_engine_t *be; rocksdb_options_t *options = rocksdb_options_create(); // Optimize RocksDB. This is the easiest way to // get RocksDB to perform well @@ -25,6 +27,9 @@ int main(int argc, char **argv) { db = rocksdb_open(options, DBPath, &err); assert(!err); + be = rocksdb_backup_engine_open(DBBackupPath, &err); + assert(!err); + // Put key-value rocksdb_writeoptions_t *writeoptions = rocksdb_writeoptions_create(); const char key[] = "key"; @@ -41,10 +46,14 @@ int main(int argc, char **argv) { assert(strcmp(returned_value, "value") == 0); free(returned_value); + rocksdb_backup_engine_create_new_backup(be, db, &err); + assert(!err); + // cleanup rocksdb_writeoptions_destroy(writeoptions); rocksdb_readoptions_destroy(readoptions); rocksdb_options_destroy(options); + rocksdb_backup_engine_close(be); rocksdb_close(db); return 0; diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index c686c90c7..9d36f8277 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -55,6 +55,7 @@ extern "C" { /* Exported types */ typedef struct rocksdb_t rocksdb_t; +typedef struct rocksdb_backup_engine_t rocksdb_backup_engine_t; typedef struct rocksdb_cache_t rocksdb_cache_t; typedef struct rocksdb_compactionfilter_t rocksdb_compactionfilter_t; typedef struct rocksdb_compactionfiltercontext_t @@ -104,6 +105,18 @@ extern rocksdb_t* rocksdb_open_for_read_only( unsigned char error_if_log_file_exist, char** errptr); +extern rocksdb_backup_engine_t* rocksdb_backup_engine_open( + const char* path, + char** errptr); + +extern void rocksdb_backup_engine_create_new_backup( + rocksdb_backup_engine_t *be, + rocksdb_t* db, + char** errptr); + +extern void rocksdb_backup_engine_close( + rocksdb_backup_engine_t *be); + extern rocksdb_t* rocksdb_open_column_families( const rocksdb_options_t* options, const char* name, From 7e50ed8c24a9ee8cb4ec5384b31865144679e13c Mon Sep 17 00:00:00 2001 From: Marko Kevac Date: Sat, 7 Feb 2015 14:25:10 +0300 Subject: [PATCH 2/5] Added some more wrappers and wrote a test for backup in C --- db/c.cc | 70 ++++++++++++++++++++++++++++++++++++++++++++- db/c_test.c | 42 +++++++++++++++++++++++++++ include/rocksdb/c.h | 44 ++++++++++++++++++++++++++-- 3 files changed, 153 insertions(+), 3 deletions(-) diff --git a/db/c.cc b/db/c.cc index c952b2e8b..f3d0fed12 100644 --- a/db/c.cc +++ b/db/c.cc @@ -72,6 +72,8 @@ using rocksdb::WriteOptions; using rocksdb::LiveFileMetaData; using rocksdb::BackupEngine; using rocksdb::BackupableDBOptions; +using rocksdb::BackupInfo; +using rocksdb::RestoreOptions; using std::shared_ptr; @@ -79,6 +81,8 @@ extern "C" { struct rocksdb_t { DB* rep; }; struct rocksdb_backup_engine_t { BackupEngine* rep; }; +struct rocksdb_backup_engine_info_t { std::vector rep; }; +struct rocksdb_restore_options_t { RestoreOptions rep; }; struct rocksdb_iterator_t { Iterator* rep; }; struct rocksdb_writebatch_t { WriteBatch rep; }; struct rocksdb_snapshot_t { const Snapshot* rep; }; @@ -532,10 +536,11 @@ rocksdb_t* rocksdb_open_for_read_only( } rocksdb_backup_engine_t* rocksdb_backup_engine_open( + const rocksdb_options_t* options, const char* path, char** errptr) { BackupEngine* be; - if (SaveError(errptr, BackupEngine::Open(Env::Default(), BackupableDBOptions(path), &be))) { + if (SaveError(errptr, BackupEngine::Open(options->rep.env, BackupableDBOptions(path), &be))) { return nullptr; } rocksdb_backup_engine_t* result = new rocksdb_backup_engine_t; @@ -550,6 +555,69 @@ void rocksdb_backup_engine_create_new_backup( SaveError(errptr, be->rep->CreateNewBackup(db->rep)); } +rocksdb_restore_options_t* rocksdb_restore_options_create() { + return new rocksdb_restore_options_t; +} + +void rocksdb_restore_options_destroy(rocksdb_restore_options_t* opt) { + delete opt; +} + +void rocksdb_restore_options_set_keep_log_files( + rocksdb_restore_options_t* opt, int v) { + opt->rep.keep_log_files = v; +} + +void rocksdb_backup_engine_restore_db_from_latest_backup( + rocksdb_backup_engine_t *be, + const char* db_dir, + const char* wal_dir, + const rocksdb_restore_options_t *restore_options, + char** errptr) { + SaveError(errptr, be->rep->RestoreDBFromLatestBackup(std::string(db_dir), std::string(wal_dir), restore_options->rep)); +} + +const rocksdb_backup_engine_info_t* rocksdb_backup_engine_get_backup_info( + rocksdb_backup_engine_t* be) { + rocksdb_backup_engine_info_t* result = new rocksdb_backup_engine_info_t; + be->rep->GetBackupInfo(&result->rep); + return result; +} + +int rocksdb_backup_engine_info_count( + const rocksdb_backup_engine_info_t* info) { + return static_cast(info->rep.size()); +} + +const int64_t rocksdb_backup_engine_info_timestamp( + const rocksdb_backup_engine_info_t* info, + int index) { + return info->rep[index].timestamp; +} + +const uint32_t rocksdb_backup_engine_info_backup_id( + const rocksdb_backup_engine_info_t* info, + int index) { + return info->rep[index].backup_id; +} + +const uint64_t rocksdb_backup_engine_info_size( + const rocksdb_backup_engine_info_t* info, + int index) { + return info->rep[index].size; +} + +const uint32_t rocksdb_backup_engine_info_number_files( + const rocksdb_backup_engine_info_t* info, + int index) { + return info->rep[index].number_files; +} + +void rocksdb_backup_engine_info_destroy( + const rocksdb_backup_engine_info_t* info) { + delete info; +} + void rocksdb_backup_engine_close( rocksdb_backup_engine_t *be) { delete be->rep; diff --git a/db/c_test.c b/db/c_test.c index ba239a43d..4f9bd4ebf 100644 --- a/db/c_test.c +++ b/db/c_test.c @@ -10,9 +10,11 @@ #include #include #include +#include const char* phase = ""; static char dbname[200]; +static char dbbackupname[200]; static void StartPhase(const char* name) { fprintf(stderr, "=== Test %s\n", name); @@ -346,6 +348,11 @@ int main(int argc, char** argv) { GetTempDir(), ((int) geteuid())); + snprintf(dbbackupname, sizeof(dbbackupname), + "%s/rocksdb_c_test-%d-backup", + GetTempDir(), + ((int) geteuid())); + StartPhase("create_objects"); cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); env = rocksdb_create_default_env(); @@ -396,6 +403,41 @@ int main(int argc, char** argv) { CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); + StartPhase("backup"); + { + rocksdb_destroy_db(options, dbbackupname, &err); + CheckNoError(err); + + rocksdb_backup_engine_t *be = rocksdb_backup_engine_open(options, dbbackupname, &err); + CheckNoError(err); + + rocksdb_backup_engine_create_new_backup(be, db, &err); + CheckNoError(err); + + rocksdb_delete(db, woptions, "foo", 3, &err); + CheckNoError(err); + + rocksdb_close(db); + + rocksdb_destroy_db(options, dbname, &err); + CheckNoError(err); + + rocksdb_restore_options_t *restore_options = rocksdb_restore_options_create(); + rocksdb_restore_options_set_keep_log_files(restore_options, 0); + rocksdb_backup_engine_restore_db_from_latest_backup(be, dbname, dbname, restore_options, &err); + CheckNoError(err); + rocksdb_restore_options_destroy(restore_options); + + rocksdb_options_set_error_if_exists(options, 0); + db = rocksdb_open(options, dbname, &err); + CheckNoError(err); + rocksdb_options_set_error_if_exists(options, 1); + + CheckGet(db, roptions, "foo", "hello"); + + rocksdb_backup_engine_close(be); + } + StartPhase("compactall"); rocksdb_compact_range(db, NULL, 0, NULL, 0); CheckGet(db, roptions, "foo", "hello"); diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index 9d36f8277..ac5f612a0 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -56,6 +56,8 @@ extern "C" { typedef struct rocksdb_t rocksdb_t; typedef struct rocksdb_backup_engine_t rocksdb_backup_engine_t; +typedef struct rocksdb_backup_engine_info_t rocksdb_backup_engine_info_t; +typedef struct rocksdb_restore_options_t rocksdb_restore_options_t; typedef struct rocksdb_cache_t rocksdb_cache_t; typedef struct rocksdb_compactionfilter_t rocksdb_compactionfilter_t; typedef struct rocksdb_compactionfiltercontext_t @@ -106,16 +108,54 @@ extern rocksdb_t* rocksdb_open_for_read_only( char** errptr); extern rocksdb_backup_engine_t* rocksdb_backup_engine_open( + const rocksdb_options_t* options, const char* path, char** errptr); extern void rocksdb_backup_engine_create_new_backup( - rocksdb_backup_engine_t *be, + rocksdb_backup_engine_t* be, rocksdb_t* db, char** errptr); +extern rocksdb_restore_options_t* rocksdb_restore_options_create(); +extern void rocksdb_restore_options_destroy(rocksdb_restore_options_t* opt); +extern void rocksdb_restore_options_set_keep_log_files( + rocksdb_restore_options_t* opt, int v); + +extern void rocksdb_backup_engine_restore_db_from_latest_backup( + rocksdb_backup_engine_t *be, + const char* db_dir, + const char* wal_dir, + const rocksdb_restore_options_t *restore_options, + char** errptr); + +extern const rocksdb_backup_engine_info_t* rocksdb_backup_engine_get_backup_info( + rocksdb_backup_engine_t* be); + +extern int rocksdb_backup_engine_info_count( + const rocksdb_backup_engine_info_t* info); + +extern const int64_t rocksdb_backup_engine_info_timestamp( + const rocksdb_backup_engine_info_t* info, + int index); + +extern const uint32_t rocksdb_backup_engine_info_backup_id( + const rocksdb_backup_engine_info_t* info, + int index); + +extern const uint64_t rocksdb_backup_engine_info_size( + const rocksdb_backup_engine_info_t* info, + int index); + +extern const uint32_t rocksdb_backup_engine_info_number_files( + const rocksdb_backup_engine_info_t* info, + int index); + +extern void rocksdb_backup_engine_info_destroy( + const rocksdb_backup_engine_info_t *info); + extern void rocksdb_backup_engine_close( - rocksdb_backup_engine_t *be); + rocksdb_backup_engine_t* be); extern rocksdb_t* rocksdb_open_column_families( const rocksdb_options_t* options, From 9651308307234b6eb08b3cdf62324b10df1e94e9 Mon Sep 17 00:00:00 2001 From: Marko Kevac Date: Mon, 9 Feb 2015 12:11:42 +0300 Subject: [PATCH 3/5] renamed backup to backup_and_restore in c_test for clarity --- db/c_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/c_test.c b/db/c_test.c index 4f9bd4ebf..b8f0ea186 100644 --- a/db/c_test.c +++ b/db/c_test.c @@ -403,7 +403,7 @@ int main(int argc, char** argv) { CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); - StartPhase("backup"); + StartPhase("backup_and_restore"); { rocksdb_destroy_db(options, dbbackupname, &err); CheckNoError(err); From d090330c8eb5b7ac61370e16305510e1e86cd913 Mon Sep 17 00:00:00 2001 From: Marko Kevac Date: Mon, 9 Feb 2015 12:16:04 +0300 Subject: [PATCH 4/5] fixed c_simple_example and added some comments --- examples/c_simple_example.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/c_simple_example.c b/examples/c_simple_example.c index 8340026db..fd79a6968 100644 --- a/examples/c_simple_example.c +++ b/examples/c_simple_example.c @@ -27,7 +27,8 @@ int main(int argc, char **argv) { db = rocksdb_open(options, DBPath, &err); assert(!err); - be = rocksdb_backup_engine_open(DBBackupPath, &err); + // open Backup Engine that we will use for backing up or database + be = rocksdb_backup_engine_open(options, DBBackupPath, &err); assert(!err); // Put key-value @@ -46,6 +47,7 @@ int main(int argc, char **argv) { assert(strcmp(returned_value, "value") == 0); free(returned_value); + // create new backup in a directory specified by DBBackupPath rocksdb_backup_engine_create_new_backup(be, db, &err); assert(!err); From 82faa377a8276faedfaeeebfbec6da6e2bc68540 Mon Sep 17 00:00:00 2001 From: Marko Kevac Date: Mon, 9 Feb 2015 19:34:50 +0300 Subject: [PATCH 5/5] added simple example for db restore from backup --- examples/c_simple_example.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/examples/c_simple_example.c b/examples/c_simple_example.c index fd79a6968..c152b7da1 100644 --- a/examples/c_simple_example.c +++ b/examples/c_simple_example.c @@ -51,6 +51,17 @@ int main(int argc, char **argv) { rocksdb_backup_engine_create_new_backup(be, db, &err); assert(!err); + rocksdb_close(db); + + // If something is wrong, you might want to restore data from last backup + rocksdb_restore_options_t *restore_options = rocksdb_restore_options_create(); + rocksdb_backup_engine_restore_db_from_latest_backup(be, DBPath, DBPath, restore_options, &err); + assert(!err); + rocksdb_restore_options_destroy(restore_options); + + db = rocksdb_open(options, DBPath, &err); + assert(!err); + // cleanup rocksdb_writeoptions_destroy(writeoptions); rocksdb_readoptions_destroy(readoptions);