NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

expose IngestExternalFile to c abi

Browse Source 
Summary:
IngestExternalFile is very useful when doing bulk load. This pr expose this API to c so many bindings can benefit from it too.
Closes https://github.com/facebook/rocksdb/pull/1454

Differential Revision: D4113420

Pulled By: yiwu-arbug

fbshipit-source-id: 307c6ae
main
Jay Lee 8 years ago committed by Facebook Github Bot
parent ce22ea99a0
commit 16fb04434f
3 changed files with 222 additions and 5 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 107
      db/c.cc
  2. 65
      db/c_test.c
  3. 53
      include/rocksdb/c.h

107
db/c.cc
Unescape View File

@ -47,10 +47,12 @@ using rocksdb::WALRecoveryMode;
using rocksdb::DB;
using rocksdb::DBOptions;
using rocksdb::Env;
using rocksdb::EnvOptions;
using rocksdb::InfoLogLevel;
using rocksdb::FileLock;
using rocksdb::FilterPolicy;
using rocksdb::FlushOptions;
using rocksdb::IngestExternalFileOptions;
using rocksdb::Iterator;
using rocksdb::Logger;
using rocksdb::MergeOperator;
@ -68,6 +70,7 @@ using rocksdb::Slice;
using rocksdb::SliceParts;
using rocksdb::SliceTransform;
using rocksdb::Snapshot;
using rocksdb::SstFileWriter;
using rocksdb::Status;
using rocksdb::WritableFile;
using rocksdb::WriteBatch;
@ -108,6 +111,9 @@ struct rocksdb_logger_t { shared_ptr<Logger> rep; };
struct rocksdb_cache_t { shared_ptr<Cache> rep; };
struct rocksdb_livefiles_t { std::vector<LiveFileMetaData> rep; };
struct rocksdb_column_family_handle_t { ColumnFamilyHandle* rep; };
struct rocksdb_envoptions_t { EnvOptions rep; };
struct rocksdb_ingestexternalfileoptions_t { IngestExternalFileOptions rep; };
struct rocksdb_sstfilewriter_t { SstFileWriter* rep; };
struct rocksdb_compactionfiltercontext_t {
CompactionFilter::Context rep;
@ -2187,6 +2193,107 @@ void rocksdb_env_destroy(rocksdb_env_t* env) {
delete env;
}
rocksdb_envoptions_t* rocksdb_envoptions_create() {
rocksdb_envoptions_t* opt = new rocksdb_envoptions_t;
return opt;
}
void rocksdb_envoptions_destroy(rocksdb_envoptions_t* opt) { delete opt; }
rocksdb_sstfilewriter_t* rocksdb_sstfilewriter_create(
const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options) {
rocksdb_sstfilewriter_t* writer = new rocksdb_sstfilewriter_t;
writer->rep =
new SstFileWriter(env->rep, io_options->rep, io_options->rep.comparator);
return writer;
}
rocksdb_sstfilewriter_t* rocksdb_sstfilewriter_create_with_comparator(
const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options,
const rocksdb_comparator_t* comparator) {
rocksdb_sstfilewriter_t* writer = new rocksdb_sstfilewriter_t;
writer->rep =
new SstFileWriter(env->rep, io_options->rep, io_options->rep.comparator);
return writer;
}
void rocksdb_sstfilewriter_open(rocksdb_sstfilewriter_t* writer,
const char* name, char** errptr) {
SaveError(errptr, writer->rep->Open(std::string(name)));
}
void rocksdb_sstfilewriter_add(rocksdb_sstfilewriter_t* writer, const char* key,
size_t keylen, const char* val, size_t vallen,
char** errptr) {
SaveError(errptr, writer->rep->Add(Slice(key, keylen), Slice(val, vallen)));
}
void rocksdb_sstfilewriter_finish(rocksdb_sstfilewriter_t* writer,
char** errptr) {
SaveError(errptr, writer->rep->Finish(NULL));
}
void rocksdb_sstfilewriter_destroy(rocksdb_sstfilewriter_t* writer) {
delete writer->rep;
delete writer;
}
rocksdb_ingestexternalfileoptions_t*
rocksdb_ingestexternalfileoptions_create() {
rocksdb_ingestexternalfileoptions_t* opt =
new rocksdb_ingestexternalfileoptions_t;
return opt;
}
void rocksdb_ingestexternalfileoptions_set_move_files(
rocksdb_ingestexternalfileoptions_t* opt, unsigned char move_files) {
opt->rep.move_files = move_files;
}
void rocksdb_ingestexternalfileoptions_set_snapshot_consistency(
rocksdb_ingestexternalfileoptions_t* opt,
unsigned char snapshot_consistency) {
opt->rep.snapshot_consistency = snapshot_consistency;
}
void rocksdb_ingestexternalfileoptions_set_allow_global_seqno(
rocksdb_ingestexternalfileoptions_t* opt,
unsigned char allow_global_seqno) {
opt->rep.allow_global_seqno = allow_global_seqno;
}
void rocksdb_ingestexternalfileoptions_set_allow_blocking_flush(
rocksdb_ingestexternalfileoptions_t* opt,
unsigned char allow_blocking_flush) {
opt->rep.allow_blocking_flush = allow_blocking_flush;
}
void rocksdb_ingestexternalfileoptions_destroy(
rocksdb_ingestexternalfileoptions_t* opt) {
delete opt;
}
void rocksdb_ingest_external_file(
rocksdb_t* db, const char* const* file_list, const size_t list_len,
const rocksdb_ingestexternalfileoptions_t* opt, char** errptr) {
std::vector<std::string> files(list_len);
for (size_t i = 0; i < list_len; ++i) {
files[i] = std::string(file_list[i]);
}
SaveError(errptr, db->rep->IngestExternalFile(files, opt->rep));
}
void rocksdb_ingest_external_file_cf(
rocksdb_t* db, rocksdb_column_family_handle_t* handle,
const char* const* file_list, const size_t list_len,
const rocksdb_ingestexternalfileoptions_t* opt, char** errptr) {
std::vector<std::string> files(list_len);
for (size_t i = 0; i < list_len; ++i) {
files[i] = std::string(file_list[i]);
}
SaveError(errptr, db->rep->IngestExternalFile(handle->rep, files, opt->rep));
}
rocksdb_slicetransform_t* rocksdb_slicetransform_create(
void* state,
void (*destructor)(void*),

65
db/c_test.c
Unescape View File

@ -12,7 +12,7 @@
#include <string.h>
#include <sys/types.h>
#ifndef OS_WIN
# include <unistd.h>
#include <unistd.h>
#endif
#include <inttypes.h>
@ -21,11 +21,10 @@
#include <Windows.h>
# define snprintf _snprintf
#define snprintf _snprintf
// Ok for uniqueness
int geteuid() {
int result = 0;
result = ((int)GetCurrentProcessId() << 16);
@ -38,13 +37,13 @@ int geteuid() {
const char* phase = "";
static char dbname[200];
static char sstfilename[200];
static char dbbackupname[200];
static void StartPhase(const char* name) {
fprintf(stderr, "=== Test %s\n", name);
phase = name;
}
static const char* GetTempDir(void) {
const char* ret = getenv("TEST_TMPDIR");
if (ret == NULL || ret[0] == '\0')
@ -304,6 +303,11 @@ int main(int argc, char** argv) {
GetTempDir(),
((int) geteuid()));
snprintf(sstfilename, sizeof(sstfilename),
"%s/rocksdb_c_test-%d-sst",
GetTempDir(),
((int)geteuid()));
StartPhase("create_objects");
cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
env = rocksdb_create_default_env();
@ -565,6 +569,59 @@ int main(int argc, char** argv) {
rocksdb_release_snapshot(db, snap);
}
StartPhase("addfile");
{
rocksdb_envoptions_t* env_opt = rocksdb_envoptions_create();
rocksdb_options_t* io_options = rocksdb_options_create();
rocksdb_sstfilewriter_t* writer =
rocksdb_sstfilewriter_create(env_opt, io_options);
unlink(sstfilename);
rocksdb_sstfilewriter_open(writer, sstfilename, &err);
CheckNoError(err);
rocksdb_sstfilewriter_add(writer, "sstk1", 5, "v1", 2, &err);
CheckNoError(err);
rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v2", 2, &err);
CheckNoError(err);
rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v3", 2, &err);
CheckNoError(err);
rocksdb_sstfilewriter_finish(writer, &err);
CheckNoError(err);
rocksdb_ingestexternalfileoptions_t* ing_opt =
rocksdb_ingestexternalfileoptions_create();
const char* file_list[1] = {sstfilename};
rocksdb_ingest_external_file(db, file_list, 1, ing_opt, &err);
CheckNoError(err);
CheckGet(db, roptions, "sstk1", "v1");
CheckGet(db, roptions, "sstk2", "v2");
CheckGet(db, roptions, "sstk3", "v3");
unlink(sstfilename);
rocksdb_sstfilewriter_open(writer, sstfilename, &err);
CheckNoError(err);
rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v4", 2, &err);
CheckNoError(err);
rocksdb_sstfilewriter_add(writer, "sstk22", 6, "v5", 2, &err);
CheckNoError(err);
rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v6", 2, &err);
CheckNoError(err);
rocksdb_sstfilewriter_finish(writer, &err);
CheckNoError(err);
rocksdb_ingest_external_file(db, file_list, 1, ing_opt, &err);
CheckNoError(err);
CheckGet(db, roptions, "sstk1", "v1");
CheckGet(db, roptions, "sstk2", "v4");
CheckGet(db, roptions, "sstk22", "v5");
CheckGet(db, roptions, "sstk3", "v6");
rocksdb_ingestexternalfileoptions_destroy(ing_opt);
rocksdb_sstfilewriter_destroy(writer);
rocksdb_options_destroy(io_options);
rocksdb_envoptions_destroy(env_opt);
}
StartPhase("repair");
{
// If we do not compact here, then the lazy deletion of

53
include/rocksdb/c.h
Unescape View File

@ -105,6 +105,9 @@ typedef struct rocksdb_writeoptions_t rocksdb_writeoptions_t;
typedef struct rocksdb_universal_compaction_options_t rocksdb_universal_compaction_options_t;
typedef struct rocksdb_livefiles_t rocksdb_livefiles_t;
typedef struct rocksdb_column_family_handle_t rocksdb_column_family_handle_t;
typedef struct rocksdb_envoptions_t rocksdb_envoptions_t;
typedef struct rocksdb_ingestexternalfileoptions_t rocksdb_ingestexternalfileoptions_t;
typedef struct rocksdb_sstfilewriter_t rocksdb_sstfilewriter_t;
/* DB operations */
@ -865,6 +868,56 @@ extern ROCKSDB_LIBRARY_API void rocksdb_env_join_all_threads(
rocksdb_env_t* env);
extern ROCKSDB_LIBRARY_API void rocksdb_env_destroy(rocksdb_env_t*);
extern ROCKSDB_LIBRARY_API rocksdb_envoptions_t* rocksdb_envoptions_create();
extern ROCKSDB_LIBRARY_API void rocksdb_envoptions_destroy(
rocksdb_envoptions_t* opt);
/* SstFile */
extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t*
rocksdb_sstfilewriter_create(const rocksdb_envoptions_t* env,
const rocksdb_options_t* io_options);
extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t*
rocksdb_sstfilewriter_create_with_comparator(
const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options,
const rocksdb_comparator_t* comparator);
extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_open(
rocksdb_sstfilewriter_t* writer, const char* name, char** errptr);
extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_add(
rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen,
const char* val, size_t vallen, char** errptr);
extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_finish(
rocksdb_sstfilewriter_t* writer, char** errptr);
extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_destroy(
rocksdb_sstfilewriter_t* writer);
extern ROCKSDB_LIBRARY_API rocksdb_ingestexternalfileoptions_t*
rocksdb_ingestexternalfileoptions_create();
extern ROCKSDB_LIBRARY_API void
rocksdb_ingestexternalfileoptions_set_move_files(
rocksdb_ingestexternalfileoptions_t* opt, unsigned char move_files);
extern ROCKSDB_LIBRARY_API void
rocksdb_ingestexternalfileoptions_set_snapshot_consistency(
rocksdb_ingestexternalfileoptions_t* opt,
unsigned char snapshot_consistency);
extern ROCKSDB_LIBRARY_API void
rocksdb_ingestexternalfileoptions_set_allow_global_seqno(
rocksdb_ingestexternalfileoptions_t* opt, unsigned char allow_global_seqno);
extern ROCKSDB_LIBRARY_API void
rocksdb_ingestexternalfileoptions_set_allow_blocking_flush(
rocksdb_ingestexternalfileoptions_t* opt,
unsigned char allow_blocking_flush);
extern ROCKSDB_LIBRARY_API void rocksdb_ingestexternalfileoptions_destroy(
rocksdb_ingestexternalfileoptions_t* opt);
extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file(
rocksdb_t* db, const char* const* file_list, const size_t list_len,
const rocksdb_ingestexternalfileoptions_t* opt, char** errptr);
extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file_cf(
rocksdb_t* db, rocksdb_column_family_handle_t* handle,
const char* const* file_list, const size_t list_len,
const rocksdb_ingestexternalfileoptions_t* opt, char** errptr);
/* SliceTransform */
extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t*

Write Preview
Loading…
Cancel
Save