From 3ac07a12fe98a6f0ea999d7303ab964e4676ff81 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 17 Sep 2020 15:39:25 -0700 Subject: [PATCH] RocksJava - Add errorIfLogFileExists parameter to RocksDB.openReadOnly (#7046) Summary: Expose from C++ API to Java API. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7046 Reviewed By: riversand963 Differential Revision: D23726297 Pulled By: pdillinger fbshipit-source-id: fc66bf626ce6fe9797e7d021ac849eacab91bf6d --- db/c.cc | 20 ++-- db/db_impl/db_impl.h | 4 +- db/db_impl/db_impl_open.cc | 14 +-- db/db_impl/db_impl_readonly.cc | 14 +-- db/db_impl/db_impl_readonly.h | 2 +- db/db_impl/db_impl_secondary.cc | 4 +- db/db_impl/db_impl_secondary.h | 4 +- include/rocksdb/c.h | 4 +- include/rocksdb/db.h | 4 +- java/rocksjni/rocksjni.cc | 40 ++++--- java/src/main/java/org/rocksdb/RocksDB.java | 100 +++++++++++++----- .../test/java/org/rocksdb/ReadOnlyTest.java | 27 +++++ 12 files changed, 163 insertions(+), 74 deletions(-) diff --git a/db/c.cc b/db/c.cc index cac977ee9..e196d0177 100644 --- a/db/c.cc +++ b/db/c.cc @@ -504,13 +504,13 @@ rocksdb_t* rocksdb_open_with_ttl( return result; } -rocksdb_t* rocksdb_open_for_read_only( - const rocksdb_options_t* options, - const char* name, - unsigned char error_if_log_file_exist, - char** errptr) { +rocksdb_t* rocksdb_open_for_read_only(const rocksdb_options_t* options, + const char* name, + unsigned char error_if_wal_file_exists, + char** errptr) { DB* db; - if (SaveError(errptr, DB::OpenForReadOnly(options->rep, std::string(name), &db, error_if_log_file_exist))) { + if (SaveError(errptr, DB::OpenForReadOnly(options->rep, std::string(name), + &db, error_if_wal_file_exists))) { return nullptr; } rocksdb_t* result = new rocksdb_t; @@ -747,7 +747,7 @@ rocksdb_t* rocksdb_open_for_read_only_column_families( int num_column_families, const char* const* column_family_names, const rocksdb_options_t* const* column_family_options, rocksdb_column_family_handle_t** column_family_handles, - unsigned char error_if_log_file_exist, char** errptr) { + unsigned char error_if_wal_file_exists, char** errptr) { std::vector column_families; for (int i = 0; i < num_column_families; i++) { column_families.push_back(ColumnFamilyDescriptor( @@ -757,8 +757,10 @@ rocksdb_t* rocksdb_open_for_read_only_column_families( DB* db; std::vector handles; - if (SaveError(errptr, DB::OpenForReadOnly(DBOptions(db_options->rep), - std::string(name), column_families, &handles, &db, error_if_log_file_exist))) { + if (SaveError(errptr, + DB::OpenForReadOnly(DBOptions(db_options->rep), + std::string(name), column_families, + &handles, &db, error_if_wal_file_exists))) { return nullptr; } diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h index 48607c7b1..30b1e9c95 100644 --- a/db/db_impl/db_impl.h +++ b/db/db_impl/db_impl.h @@ -1186,8 +1186,8 @@ class DBImpl : public DB { // skipped. virtual Status Recover( const std::vector& column_families, - bool read_only = false, bool error_if_log_file_exist = false, - bool error_if_data_exists_in_logs = false, + bool read_only = false, bool error_if_wal_file_exists = false, + bool error_if_data_exists_in_wals = false, uint64_t* recovered_seq = nullptr); virtual bool OwnTablesAndLogs() const { return true; } diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index 659ac1b7e..b3cd8dd8e 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -364,7 +364,7 @@ IOStatus Directories::SetDirectories(FileSystem* fs, const std::string& dbname, Status DBImpl::Recover( const std::vector& column_families, bool read_only, - bool error_if_log_file_exist, bool error_if_data_exists_in_logs, + bool error_if_wal_file_exists, bool error_if_data_exists_in_wals, uint64_t* recovered_seq) { mutex_.AssertHeld(); @@ -588,11 +588,11 @@ Status DBImpl::Recover( } if (logs.size() > 0) { - if (error_if_log_file_exist) { + if (error_if_wal_file_exists) { return Status::Corruption( - "The db was opened in readonly mode with error_if_log_file_exist" - "flag but a log file already exists"); - } else if (error_if_data_exists_in_logs) { + "The db was opened in readonly mode with error_if_wal_file_exists" + "flag but a WAL file already exists"); + } else if (error_if_data_exists_in_wals) { for (auto& log : logs) { std::string fname = LogFileName(immutable_db_options_.wal_dir, log); uint64_t bytes; @@ -600,8 +600,8 @@ Status DBImpl::Recover( if (s.ok()) { if (bytes > 0) { return Status::Corruption( - "error_if_data_exists_in_logs is set but there are data " - " in log files."); + "error_if_data_exists_in_wals is set but there are data " + " in WAL files."); } } } diff --git a/db/db_impl/db_impl_readonly.cc b/db/db_impl/db_impl_readonly.cc index 7c277206f..84d03863d 100644 --- a/db/db_impl/db_impl_readonly.cc +++ b/db/db_impl/db_impl_readonly.cc @@ -151,7 +151,7 @@ Status OpenForReadOnlyCheckExistence(const DBOptions& db_options, } // namespace Status DB::OpenForReadOnly(const Options& options, const std::string& dbname, - DB** dbptr, bool /*error_if_log_file_exist*/) { + DB** dbptr, bool /*error_if_wal_file_exists*/) { // If dbname does not exist in the file system, should not do anything Status s = OpenForReadOnlyCheckExistence(options, dbname); if (!s.ok()) { @@ -188,7 +188,7 @@ Status DB::OpenForReadOnly( const DBOptions& db_options, const std::string& dbname, const std::vector& column_families, std::vector* handles, DB** dbptr, - bool error_if_log_file_exist) { + bool error_if_wal_file_exists) { // If dbname does not exist in the file system, should not do anything Status s = OpenForReadOnlyCheckExistence(db_options, dbname); if (!s.ok()) { @@ -197,14 +197,14 @@ Status DB::OpenForReadOnly( return DBImplReadOnly::OpenForReadOnlyWithoutCheck( db_options, dbname, column_families, handles, dbptr, - error_if_log_file_exist); + error_if_wal_file_exists); } Status DBImplReadOnly::OpenForReadOnlyWithoutCheck( const DBOptions& db_options, const std::string& dbname, const std::vector& column_families, std::vector* handles, DB** dbptr, - bool error_if_log_file_exist) { + bool error_if_wal_file_exists) { *dbptr = nullptr; handles->clear(); @@ -212,7 +212,7 @@ Status DBImplReadOnly::OpenForReadOnlyWithoutCheck( DBImplReadOnly* impl = new DBImplReadOnly(db_options, dbname); impl->mutex_.Lock(); Status s = impl->Recover(column_families, true /* read only */, - error_if_log_file_exist); + error_if_wal_file_exists); if (s.ok()) { // set column family handles for (auto cf : column_families) { @@ -253,7 +253,7 @@ Status DBImplReadOnly::OpenForReadOnlyWithoutCheck( Status DB::OpenForReadOnly(const Options& /*options*/, const std::string& /*dbname*/, DB** /*dbptr*/, - bool /*error_if_log_file_exist*/) { + bool /*error_if_wal_file_exists*/) { return Status::NotSupported("Not supported in ROCKSDB_LITE."); } @@ -261,7 +261,7 @@ Status DB::OpenForReadOnly( const DBOptions& /*db_options*/, const std::string& /*dbname*/, const std::vector& /*column_families*/, std::vector* /*handles*/, DB** /*dbptr*/, - bool /*error_if_log_file_exist*/) { + bool /*error_if_wal_file_exists*/) { return Status::NotSupported("Not supported in ROCKSDB_LITE."); } #endif // !ROCKSDB_LITE diff --git a/db/db_impl/db_impl_readonly.h b/db/db_impl/db_impl_readonly.h index c19a43899..090d67d0f 100644 --- a/db/db_impl/db_impl_readonly.h +++ b/db/db_impl/db_impl_readonly.h @@ -138,7 +138,7 @@ class DBImplReadOnly : public DBImpl { const DBOptions& db_options, const std::string& dbname, const std::vector& column_families, std::vector* handles, DB** dbptr, - bool error_if_log_file_exist = false); + bool error_if_wal_file_exists = false); friend class DB; }; } // namespace ROCKSDB_NAMESPACE diff --git a/db/db_impl/db_impl_secondary.cc b/db/db_impl/db_impl_secondary.cc index 63fb1c1db..ce04f0663 100644 --- a/db/db_impl/db_impl_secondary.cc +++ b/db/db_impl/db_impl_secondary.cc @@ -28,8 +28,8 @@ DBImplSecondary::~DBImplSecondary() {} Status DBImplSecondary::Recover( const std::vector& column_families, - bool /*readonly*/, bool /*error_if_log_file_exist*/, - bool /*error_if_data_exists_in_logs*/, uint64_t*) { + bool /*readonly*/, bool /*error_if_wal_file_exists*/, + bool /*error_if_data_exists_in_wals*/, uint64_t*) { mutex_.AssertHeld(); JobContext job_context(0); diff --git a/db/db_impl/db_impl_secondary.h b/db/db_impl/db_impl_secondary.h index 24f2e7767..8fc58616f 100644 --- a/db/db_impl/db_impl_secondary.h +++ b/db/db_impl/db_impl_secondary.h @@ -77,8 +77,8 @@ class DBImplSecondary : public DBImpl { // Recover by replaying MANIFEST and WAL. Also initialize manifest_reader_ // and log_readers_ to facilitate future operations. Status Recover(const std::vector& column_families, - bool read_only, bool error_if_log_file_exist, - bool error_if_data_exists_in_logs, + bool read_only, bool error_if_wal_file_exists, + bool error_if_data_exists_in_wals, uint64_t* = nullptr) override; // Implementations of the DB interface diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index 74ff9349b..b8c72ae75 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -136,7 +136,7 @@ extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_with_ttl( extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_for_read_only( const rocksdb_options_t* options, const char* name, - unsigned char error_if_log_file_exist, char** errptr); + unsigned char error_if_wal_file_exists, char** errptr); extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary( const rocksdb_options_t* options, const char* name, @@ -232,7 +232,7 @@ rocksdb_open_for_read_only_column_families( const char* const* column_family_names, const rocksdb_options_t* const* column_family_options, rocksdb_column_family_handle_t** column_family_handles, - unsigned char error_if_log_file_exist, char** errptr); + unsigned char error_if_wal_file_exists, char** errptr); extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary_column_families( const rocksdb_options_t* options, const char* name, diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index d6b02ab74..7a6e06b92 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -157,7 +157,7 @@ class DB { // return Status::NotSupported. static Status OpenForReadOnly(const Options& options, const std::string& name, DB** dbptr, - bool error_if_log_file_exist = false); + bool error_if_wal_file_exists = false); // Open the database for read only with column families. When opening DB with // read only, you can specify only a subset of column families in the @@ -171,7 +171,7 @@ class DB { const DBOptions& db_options, const std::string& name, const std::vector& column_families, std::vector* handles, DB** dbptr, - bool error_if_log_file_exist = false); + bool error_if_wal_file_exists = false); // The following OpenAsSecondary functions create a secondary instance that // can dynamically tail the MANIFEST of a primary that must have already been diff --git a/java/rocksjni/rocksjni.cc b/java/rocksjni/rocksjni.cc index 5221f1ed3..0f463e9b3 100644 --- a/java/rocksjni/rocksjni.cc +++ b/java/rocksjni/rocksjni.cc @@ -72,15 +72,19 @@ jlong Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2( /* * Class: org_rocksdb_RocksDB * Method: openROnly - * Signature: (JLjava/lang/String;)J + * Signature: (JLjava/lang/String;Z)J */ -jlong Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2( - JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path) { +jlong Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2Z( + JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, + jboolean jerror_if_wal_file_exists) { + const bool error_if_wal_file_exists = jerror_if_wal_file_exists == JNI_TRUE; return rocksdb_open_helper( env, jopt_handle, jdb_path, - [](const ROCKSDB_NAMESPACE::Options& options, const std::string& db_path, - ROCKSDB_NAMESPACE::DB** db) { - return ROCKSDB_NAMESPACE::DB::OpenForReadOnly(options, db_path, db); + [error_if_wal_file_exists](const ROCKSDB_NAMESPACE::Options& options, + const std::string& db_path, + ROCKSDB_NAMESPACE::DB** db) { + return ROCKSDB_NAMESPACE::DB::OpenForReadOnly(options, db_path, db, + error_if_wal_file_exists); }); } @@ -172,21 +176,25 @@ jlongArray rocksdb_open_helper( /* * Class: org_rocksdb_RocksDB * Method: openROnly - * Signature: (JLjava/lang/String;[[B[J)[J + * Signature: (JLjava/lang/String;[[B[JZ)[J */ -jlongArray Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2_3_3B_3J( +jlongArray Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2_3_3B_3JZ( JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path, - jobjectArray jcolumn_names, jlongArray jcolumn_options) { + jobjectArray jcolumn_names, jlongArray jcolumn_options, + jboolean jerror_if_wal_file_exists) { + const bool error_if_wal_file_exists = jerror_if_wal_file_exists == JNI_TRUE; return rocksdb_open_helper( env, jopt_handle, jdb_path, jcolumn_names, jcolumn_options, - [](const ROCKSDB_NAMESPACE::DBOptions& options, - const std::string& db_path, - const std::vector& - column_families, - std::vector* handles, - ROCKSDB_NAMESPACE::DB** db) { + [error_if_wal_file_exists]( + const ROCKSDB_NAMESPACE::DBOptions& options, + const std::string& db_path, + const std::vector& + column_families, + std::vector* handles, + ROCKSDB_NAMESPACE::DB** db) { return ROCKSDB_NAMESPACE::DB::OpenForReadOnly( - options, db_path, column_families, handles, db); + options, db_path, column_families, handles, db, + error_if_wal_file_exists); }); } diff --git a/java/src/main/java/org/rocksdb/RocksDB.java b/java/src/main/java/org/rocksdb/RocksDB.java index a66c8077c..471a9bbd9 100644 --- a/java/src/main/java/org/rocksdb/RocksDB.java +++ b/java/src/main/java/org/rocksdb/RocksDB.java @@ -329,10 +329,61 @@ public class RocksDB extends RocksObject { throws RocksDBException { // This allows to use the rocksjni default Options instead of // the c++ one. - Options options = new Options(); + final Options options = new Options(); return openReadOnly(options, path); } + /** + * The factory constructor of RocksDB that opens a RocksDB instance in + * Read-Only mode given the path to the database using the specified + * options and db path. + * + * Options instance *should* not be disposed before all DBs using this options + * instance have been closed. If user doesn't call options dispose explicitly, + * then this options instance will be GC'd automatically. + * + * @param options {@link Options} instance. + * @param path the path to the RocksDB. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static RocksDB openReadOnly(final Options options, final String path) + throws RocksDBException { + return openReadOnly(options, path, false); + } + + /** + * The factory constructor of RocksDB that opens a RocksDB instance in + * Read-Only mode given the path to the database using the specified + * options and db path. + * + * Options instance *should* not be disposed before all DBs using this options + * instance have been closed. If user doesn't call options dispose explicitly, + * then this options instance will be GC'd automatically. + * + * @param options {@link Options} instance. + * @param path the path to the RocksDB. + * @param errorIfWalFileExists true to raise an error when opening the db + * if a Write Ahead Log file exists, false otherwise. + * @return a {@link RocksDB} instance on success, null if the specified + * {@link RocksDB} can not be opened. + * + * @throws RocksDBException thrown if error happens in underlying + * native library. + */ + public static RocksDB openReadOnly(final Options options, final String path, + final boolean errorIfWalFileExists) throws RocksDBException { + // when non-default Options is used, keeping an Options reference + // in RocksDB can prevent Java to GC during the life-time of + // the currently-created RocksDB. + final RocksDB db = new RocksDB(openROnly(options.nativeHandle_, path, errorIfWalFileExists)); + db.storeOptionsInstance(options); + return db; + } + /** * The factory constructor of RocksDB that opens a RocksDB instance in * Read-Only mode given the path to the database using the default @@ -355,8 +406,7 @@ public class RocksDB extends RocksObject { // This allows to use the rocksjni default Options instead of // the c++ one. final DBOptions options = new DBOptions(); - return openReadOnly(options, path, columnFamilyDescriptors, - columnFamilyHandles); + return openReadOnly(options, path, columnFamilyDescriptors, columnFamilyHandles, false); } /** @@ -364,26 +414,27 @@ public class RocksDB extends RocksObject { * Read-Only mode given the path to the database using the specified * options and db path. * - * Options instance *should* not be disposed before all DBs using this options - * instance have been closed. If user doesn't call options dispose explicitly, - * then this options instance will be GC'd automatically. + *

This open method allows to open RocksDB using a subset of available + * column families

+ *

Options instance *should* not be disposed before all DBs using this + * options instance have been closed. If user doesn't call options dispose + * explicitly,then this options instance will be GC'd automatically.

* - * @param options {@link Options} instance. + * @param options {@link DBOptions} instance. * @param path the path to the RocksDB. + * @param columnFamilyDescriptors list of column family descriptors + * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances + * on open. * @return a {@link RocksDB} instance on success, null if the specified * {@link RocksDB} can not be opened. * * @throws RocksDBException thrown if error happens in underlying * native library. */ - public static RocksDB openReadOnly(final Options options, final String path) - throws RocksDBException { - // when non-default Options is used, keeping an Options reference - // in RocksDB can prevent Java to GC during the life-time of - // the currently-created RocksDB. - final RocksDB db = new RocksDB(openROnly(options.nativeHandle_, path)); - db.storeOptionsInstance(options); - return db; + public static RocksDB openReadOnly(final DBOptions options, final String path, + final List columnFamilyDescriptors, + final List columnFamilyHandles) throws RocksDBException { + return openReadOnly(options, path, columnFamilyDescriptors, columnFamilyHandles, false); } /** @@ -402,6 +453,8 @@ public class RocksDB extends RocksObject { * @param columnFamilyDescriptors list of column family descriptors * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances * on open. + * @param errorIfWalFileExists true to raise an error when opening the db + * if a Write Ahead Log file exists, false otherwise. * @return a {@link RocksDB} instance on success, null if the specified * {@link RocksDB} can not be opened. * @@ -410,7 +463,7 @@ public class RocksDB extends RocksObject { */ public static RocksDB openReadOnly(final DBOptions options, final String path, final List columnFamilyDescriptors, - final List columnFamilyHandles) + final List columnFamilyHandles, final boolean errorIfWalFileExists) throws RocksDBException { // when non-default Options is used, keeping an Options reference // in RocksDB can prevent Java to GC during the life-time of @@ -425,8 +478,8 @@ public class RocksDB extends RocksObject { cfOptionHandles[i] = cfDescriptor.getOptions().nativeHandle_; } - final long[] handles = openROnly(options.nativeHandle_, path, cfNames, - cfOptionHandles); + final long[] handles = + openROnly(options.nativeHandle_, path, cfNames, cfOptionHandles, errorIfWalFileExists); final RocksDB db = new RocksDB(handles[0]); db.storeOptionsInstance(options); @@ -4381,8 +4434,8 @@ public class RocksDB extends RocksObject { final String path, final byte[][] columnFamilyNames, final long[] columnFamilyOptions) throws RocksDBException; - private native static long openROnly(final long optionsHandle, - final String path) throws RocksDBException; + private native static long openROnly(final long optionsHandle, final String path, + final boolean errorIfWalFileExists) throws RocksDBException; /** * @param optionsHandle Native handle pointing to an Options object @@ -4396,10 +4449,9 @@ public class RocksDB extends RocksObject { * * @throws RocksDBException thrown if the database could not be opened */ - private native static long[] openROnly(final long optionsHandle, - final String path, final byte[][] columnFamilyNames, - final long[] columnFamilyOptions - ) throws RocksDBException; + private native static long[] openROnly(final long optionsHandle, final String path, + final byte[][] columnFamilyNames, final long[] columnFamilyOptions, + final boolean errorIfWalFileExists) throws RocksDBException; private native static long openAsSecondary(final long optionsHandle, final String path, final String secondaryPath) throws RocksDBException; diff --git a/java/src/test/java/org/rocksdb/ReadOnlyTest.java b/java/src/test/java/org/rocksdb/ReadOnlyTest.java index 6d5bc96fc..ad6e746aa 100644 --- a/java/src/test/java/org/rocksdb/ReadOnlyTest.java +++ b/java/src/test/java/org/rocksdb/ReadOnlyTest.java @@ -302,4 +302,31 @@ public class ReadOnlyTest { } } } + + @Test(expected = RocksDBException.class) + public void errorIfWalFileExists() throws RocksDBException { + try (final Options options = new Options().setCreateIfMissing(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + // no-op + } + + try (final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()) { + final List cfDescriptors = + Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts)); + + final List readOnlyColumnFamilyHandleList = new ArrayList<>(); + try (final DBOptions options = new DBOptions(); + final RocksDB rDb = RocksDB.openReadOnly(options, dbFolder.getRoot().getAbsolutePath(), + cfDescriptors, readOnlyColumnFamilyHandleList, true);) { + try { + // no-op... should have raised an error as errorIfWalFileExists=true + + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : readOnlyColumnFamilyHandleList) { + columnFamilyHandle.close(); + } + } + } + } + } }