diff --git a/db/c.cc b/db/c.cc index 89b392b59..f2503cc50 100644 --- a/db/c.cc +++ b/db/c.cc @@ -47,10 +47,12 @@ using rocksdb::WALRecoveryMode; using rocksdb::DB; using rocksdb::DBOptions; using rocksdb::Env; +using rocksdb::EnvOptions; using rocksdb::InfoLogLevel; using rocksdb::FileLock; using rocksdb::FilterPolicy; using rocksdb::FlushOptions; +using rocksdb::IngestExternalFileOptions; using rocksdb::Iterator; using rocksdb::Logger; using rocksdb::MergeOperator; @@ -68,6 +70,7 @@ using rocksdb::Slice; using rocksdb::SliceParts; using rocksdb::SliceTransform; using rocksdb::Snapshot; +using rocksdb::SstFileWriter; using rocksdb::Status; using rocksdb::WritableFile; using rocksdb::WriteBatch; @@ -108,6 +111,9 @@ struct rocksdb_logger_t { shared_ptr rep; }; struct rocksdb_cache_t { shared_ptr rep; }; struct rocksdb_livefiles_t { std::vector rep; }; struct rocksdb_column_family_handle_t { ColumnFamilyHandle* rep; }; +struct rocksdb_envoptions_t { EnvOptions rep; }; +struct rocksdb_ingestexternalfileoptions_t { IngestExternalFileOptions rep; }; +struct rocksdb_sstfilewriter_t { SstFileWriter* rep; }; struct rocksdb_compactionfiltercontext_t { CompactionFilter::Context rep; @@ -2187,6 +2193,107 @@ void rocksdb_env_destroy(rocksdb_env_t* env) { delete env; } +rocksdb_envoptions_t* rocksdb_envoptions_create() { + rocksdb_envoptions_t* opt = new rocksdb_envoptions_t; + return opt; +} + +void rocksdb_envoptions_destroy(rocksdb_envoptions_t* opt) { delete opt; } + +rocksdb_sstfilewriter_t* rocksdb_sstfilewriter_create( + const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options) { + rocksdb_sstfilewriter_t* writer = new rocksdb_sstfilewriter_t; + writer->rep = + new SstFileWriter(env->rep, io_options->rep, io_options->rep.comparator); + return writer; +} + +rocksdb_sstfilewriter_t* rocksdb_sstfilewriter_create_with_comparator( + const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options, + const rocksdb_comparator_t* comparator) { + rocksdb_sstfilewriter_t* writer = new rocksdb_sstfilewriter_t; + writer->rep = + new SstFileWriter(env->rep, io_options->rep, io_options->rep.comparator); + return writer; +} + +void rocksdb_sstfilewriter_open(rocksdb_sstfilewriter_t* writer, + const char* name, char** errptr) { + SaveError(errptr, writer->rep->Open(std::string(name))); +} + +void rocksdb_sstfilewriter_add(rocksdb_sstfilewriter_t* writer, const char* key, + size_t keylen, const char* val, size_t vallen, + char** errptr) { + SaveError(errptr, writer->rep->Add(Slice(key, keylen), Slice(val, vallen))); +} + +void rocksdb_sstfilewriter_finish(rocksdb_sstfilewriter_t* writer, + char** errptr) { + SaveError(errptr, writer->rep->Finish(NULL)); +} + +void rocksdb_sstfilewriter_destroy(rocksdb_sstfilewriter_t* writer) { + delete writer->rep; + delete writer; +} + +rocksdb_ingestexternalfileoptions_t* +rocksdb_ingestexternalfileoptions_create() { + rocksdb_ingestexternalfileoptions_t* opt = + new rocksdb_ingestexternalfileoptions_t; + return opt; +} + +void rocksdb_ingestexternalfileoptions_set_move_files( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char move_files) { + opt->rep.move_files = move_files; +} + +void rocksdb_ingestexternalfileoptions_set_snapshot_consistency( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char snapshot_consistency) { + opt->rep.snapshot_consistency = snapshot_consistency; +} + +void rocksdb_ingestexternalfileoptions_set_allow_global_seqno( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char allow_global_seqno) { + opt->rep.allow_global_seqno = allow_global_seqno; +} + +void rocksdb_ingestexternalfileoptions_set_allow_blocking_flush( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char allow_blocking_flush) { + opt->rep.allow_blocking_flush = allow_blocking_flush; +} + +void rocksdb_ingestexternalfileoptions_destroy( + rocksdb_ingestexternalfileoptions_t* opt) { + delete opt; +} + +void rocksdb_ingest_external_file( + rocksdb_t* db, const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr) { + std::vector files(list_len); + for (size_t i = 0; i < list_len; ++i) { + files[i] = std::string(file_list[i]); + } + SaveError(errptr, db->rep->IngestExternalFile(files, opt->rep)); +} + +void rocksdb_ingest_external_file_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, + const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr) { + std::vector files(list_len); + for (size_t i = 0; i < list_len; ++i) { + files[i] = std::string(file_list[i]); + } + SaveError(errptr, db->rep->IngestExternalFile(handle->rep, files, opt->rep)); +} + rocksdb_slicetransform_t* rocksdb_slicetransform_create( void* state, void (*destructor)(void*), diff --git a/db/c_test.c b/db/c_test.c index 0dc3a974c..82339a1fa 100644 --- a/db/c_test.c +++ b/db/c_test.c @@ -12,7 +12,7 @@ #include #include #ifndef OS_WIN -# include +#include #endif #include @@ -21,11 +21,10 @@ #include -# define snprintf _snprintf +#define snprintf _snprintf // Ok for uniqueness int geteuid() { - int result = 0; result = ((int)GetCurrentProcessId() << 16); @@ -38,13 +37,13 @@ int geteuid() { const char* phase = ""; static char dbname[200]; +static char sstfilename[200]; static char dbbackupname[200]; static void StartPhase(const char* name) { fprintf(stderr, "=== Test %s\n", name); phase = name; } - static const char* GetTempDir(void) { const char* ret = getenv("TEST_TMPDIR"); if (ret == NULL || ret[0] == '\0') @@ -304,6 +303,11 @@ int main(int argc, char** argv) { GetTempDir(), ((int) geteuid())); + snprintf(sstfilename, sizeof(sstfilename), + "%s/rocksdb_c_test-%d-sst", + GetTempDir(), + ((int)geteuid())); + StartPhase("create_objects"); cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); env = rocksdb_create_default_env(); @@ -565,6 +569,59 @@ int main(int argc, char** argv) { rocksdb_release_snapshot(db, snap); } + StartPhase("addfile"); + { + rocksdb_envoptions_t* env_opt = rocksdb_envoptions_create(); + rocksdb_options_t* io_options = rocksdb_options_create(); + rocksdb_sstfilewriter_t* writer = + rocksdb_sstfilewriter_create(env_opt, io_options); + + unlink(sstfilename); + rocksdb_sstfilewriter_open(writer, sstfilename, &err); + CheckNoError(err); + rocksdb_sstfilewriter_add(writer, "sstk1", 5, "v1", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v2", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v3", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_finish(writer, &err); + CheckNoError(err); + + rocksdb_ingestexternalfileoptions_t* ing_opt = + rocksdb_ingestexternalfileoptions_create(); + const char* file_list[1] = {sstfilename}; + rocksdb_ingest_external_file(db, file_list, 1, ing_opt, &err); + CheckNoError(err); + CheckGet(db, roptions, "sstk1", "v1"); + CheckGet(db, roptions, "sstk2", "v2"); + CheckGet(db, roptions, "sstk3", "v3"); + + unlink(sstfilename); + rocksdb_sstfilewriter_open(writer, sstfilename, &err); + CheckNoError(err); + rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v4", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_add(writer, "sstk22", 6, "v5", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v6", 2, &err); + CheckNoError(err); + rocksdb_sstfilewriter_finish(writer, &err); + CheckNoError(err); + + rocksdb_ingest_external_file(db, file_list, 1, ing_opt, &err); + CheckNoError(err); + CheckGet(db, roptions, "sstk1", "v1"); + CheckGet(db, roptions, "sstk2", "v4"); + CheckGet(db, roptions, "sstk22", "v5"); + CheckGet(db, roptions, "sstk3", "v6"); + + rocksdb_ingestexternalfileoptions_destroy(ing_opt); + rocksdb_sstfilewriter_destroy(writer); + rocksdb_options_destroy(io_options); + rocksdb_envoptions_destroy(env_opt); + } + StartPhase("repair"); { // If we do not compact here, then the lazy deletion of @@ -700,7 +757,7 @@ int main(int argc, char** argv) { rocksdb_destroy_db(options, dbname, &err); CheckNoError(err) - rocksdb_options_t* db_options = rocksdb_options_create(); + rocksdb_options_t* db_options = rocksdb_options_create(); rocksdb_options_set_create_if_missing(db_options, 1); db = rocksdb_open(db_options, dbname, &err); CheckNoError(err) diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index 816f5220a..fe32a3ba1 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -105,6 +105,9 @@ typedef struct rocksdb_writeoptions_t rocksdb_writeoptions_t; typedef struct rocksdb_universal_compaction_options_t rocksdb_universal_compaction_options_t; typedef struct rocksdb_livefiles_t rocksdb_livefiles_t; typedef struct rocksdb_column_family_handle_t rocksdb_column_family_handle_t; +typedef struct rocksdb_envoptions_t rocksdb_envoptions_t; +typedef struct rocksdb_ingestexternalfileoptions_t rocksdb_ingestexternalfileoptions_t; +typedef struct rocksdb_sstfilewriter_t rocksdb_sstfilewriter_t; /* DB operations */ @@ -865,6 +868,56 @@ extern ROCKSDB_LIBRARY_API void rocksdb_env_join_all_threads( rocksdb_env_t* env); extern ROCKSDB_LIBRARY_API void rocksdb_env_destroy(rocksdb_env_t*); +extern ROCKSDB_LIBRARY_API rocksdb_envoptions_t* rocksdb_envoptions_create(); +extern ROCKSDB_LIBRARY_API void rocksdb_envoptions_destroy( + rocksdb_envoptions_t* opt); + +/* SstFile */ + +extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t* +rocksdb_sstfilewriter_create(const rocksdb_envoptions_t* env, + const rocksdb_options_t* io_options); +extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t* +rocksdb_sstfilewriter_create_with_comparator( + const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options, + const rocksdb_comparator_t* comparator); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_open( + rocksdb_sstfilewriter_t* writer, const char* name, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_add( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_finish( + rocksdb_sstfilewriter_t* writer, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_destroy( + rocksdb_sstfilewriter_t* writer); + +extern ROCKSDB_LIBRARY_API rocksdb_ingestexternalfileoptions_t* +rocksdb_ingestexternalfileoptions_create(); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_move_files( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char move_files); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_snapshot_consistency( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char snapshot_consistency); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_allow_global_seqno( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char allow_global_seqno); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_allow_blocking_flush( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char allow_blocking_flush); +extern ROCKSDB_LIBRARY_API void rocksdb_ingestexternalfileoptions_destroy( + rocksdb_ingestexternalfileoptions_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file( + rocksdb_t* db, const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, + const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr); + /* SliceTransform */ extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t*