Make EncryptionProvider and BlockCipher into Customizable objects (#8354)

Summary:
Made the EncryptionProvider and BlockCipher classes inherit from Customizable.  Added/fixed the CreateFromString method to these classes to create instances from builtin or registered classes.  Added tests to verify that instances can be registered and retrieved as appropriate.

Added the ability to configure the builtin (CTR, ROT13) classes from configurable properties.  Added the appropriate tests.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8354

Reviewed By: zhichao-cao

Differential Revision: D29558949

Pulled By: mrambacher

fbshipit-source-id: c20286b32d179777e060f51a58943e9b0cf81d04

db/db_test_util.cc

@@ -68,9 +68,13 @@ DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
 #ifndef ROCKSDB_LITE
   if (getenv("ENCRYPTED_ENV")) {
     std::shared_ptr<EncryptionProvider> provider;
-    Status s = EncryptionProvider::CreateFromString(
-        config_options, std::string("test://") + getenv("ENCRYPTED_ENV"),
-        &provider);
+    std::string provider_id = getenv("ENCRYPTED_ENV");
+    if (provider_id.find("=") == std::string::npos &&
+        !EndsWith(provider_id, "://test")) {
+      provider_id = provider_id + "://test";
+    }
+    EXPECT_OK(EncryptionProvider::CreateFromString(ConfigOptions(), provider_id,
+                                                   &provider));
     encrypted_env_ = NewEncryptedEnv(mem_env_ ? mem_env_ : base_env, provider);
   }
 #endif  // !ROCKSDB_LITE

env/env_basic_test.cc

@@ -17,15 +17,81 @@
 #include "test_util/testharness.h"
 
 namespace ROCKSDB_NAMESPACE {
+typedef Env* CreateEnvFunc();
+namespace {
+// These functions are used to create the various environments under which this
+// test can execute. These functions are used to allow the test cases to be
+// created without the Env being initialized, thereby eliminating a potential
+// static initialization fiasco/race condition when attempting to get a
+// custom/configured env prior to main being invoked.
+
+static Env* GetDefaultEnv() { return Env::Default(); }
+
+static Env* GetMockEnv() {
+  static std::unique_ptr<Env> mock_env(new MockEnv(Env::Default()));
+  return mock_env.get();
+}
+#ifndef ROCKSDB_LITE
+static Env* NewTestEncryptedEnv(Env* base, const std::string& provider_id) {
+  ConfigOptions config_opts;
+  config_opts.invoke_prepare_options = false;
+
+  std::shared_ptr<EncryptionProvider> provider;
+  EXPECT_OK(EncryptionProvider::CreateFromString(config_opts, provider_id,
+                                                 &provider));
+  return NewEncryptedEnv(base, provider);
+}
+
+static Env* GetCtrEncryptedEnv() {
+  static std::unique_ptr<Env> ctr_encrypt_env(
+      NewTestEncryptedEnv(Env::Default(), "CTR://test"));
+  return ctr_encrypt_env.get();
+}
+
+static Env* GetMemoryEnv() {
+  static std::unique_ptr<Env> mem_env(NewMemEnv(Env::Default()));
+  return mem_env.get();
+}
+
+static Env* GetTestEnv() {
+  static std::shared_ptr<Env> env_guard;
+  static Env* custom_env = nullptr;
+  if (custom_env == nullptr) {
+    const char* uri = getenv("TEST_ENV_URI");
+    if (uri != nullptr) {
+      EXPECT_OK(Env::CreateFromUri(ConfigOptions(), uri, "", &custom_env,
+                                   &env_guard));
+    }
+  }
+  EXPECT_NE(custom_env, nullptr);
+  return custom_env;
+}
 
-class EnvBasicTestWithParam : public testing::Test,
-                              public ::testing::WithParamInterface<Env*> {
+static Env* GetTestFS() {
+  static std::shared_ptr<Env> fs_env_guard;
+  static Env* fs_env = nullptr;
+  if (fs_env == nullptr) {
+    const char* uri = getenv("TEST_FS_URI");
+    if (uri != nullptr) {
+      EXPECT_OK(
+          Env::CreateFromUri(ConfigOptions(), uri, "", &fs_env, &fs_env_guard));
+    }
+  }
+  EXPECT_NE(fs_env, nullptr);
+  return fs_env;
+}
+#endif  // ROCKSDB_LITE
+
+}  // namespace
+class EnvBasicTestWithParam
+    : public testing::Test,
+      public ::testing::WithParamInterface<CreateEnvFunc*> {
  public:
   Env* env_;
   const EnvOptions soptions_;
   std::string test_dir_;
 
-  EnvBasicTestWithParam() : env_(GetParam()) {
+  EnvBasicTestWithParam() : env_(GetParam()()) {
     test_dir_ = test::PerThreadDBPath(env_, "env_basic_test");
   }
 
@@ -37,35 +103,22 @@ class EnvBasicTestWithParam : public testing::Test,
 class EnvMoreTestWithParam : public EnvBasicTestWithParam {};
 
 INSTANTIATE_TEST_CASE_P(EnvDefault, EnvBasicTestWithParam,
-                        ::testing::Values(Env::Default()));
+                        ::testing::Values(&GetDefaultEnv));
 INSTANTIATE_TEST_CASE_P(EnvDefault, EnvMoreTestWithParam,
-                        ::testing::Values(Env::Default()));
+                        ::testing::Values(&GetDefaultEnv));
 
-static std::unique_ptr<Env> mock_env(new MockEnv(Env::Default()));
 INSTANTIATE_TEST_CASE_P(MockEnv, EnvBasicTestWithParam,
-                        ::testing::Values(mock_env.get()));
+                        ::testing::Values(&GetMockEnv));
 
 #ifndef ROCKSDB_LITE
-static Env* NewTestEncryptedEnv(Env* base, const std::string& provider_id) {
-  std::shared_ptr<EncryptionProvider> provider;
-  EXPECT_OK(EncryptionProvider::CreateFromString(ConfigOptions(), provider_id,
-                                                 &provider));
-  return NewEncryptedEnv(base, provider);
-}
-
 // next statements run env test against default encryption code.
-static std::unique_ptr<Env> ctr_encrypt_env(NewTestEncryptedEnv(Env::Default(),
-                                                                "test://CTR"));
 INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvBasicTestWithParam,
-                        ::testing::Values(ctr_encrypt_env.get()));
+                        ::testing::Values(&GetCtrEncryptedEnv));
 INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvMoreTestWithParam,
-                        ::testing::Values(ctr_encrypt_env.get()));
-#endif  // ROCKSDB_LITE
+                        ::testing::Values(&GetCtrEncryptedEnv));
 
-#ifndef ROCKSDB_LITE
-static std::unique_ptr<Env> mem_env(NewMemEnv(Env::Default()));
 INSTANTIATE_TEST_CASE_P(MemEnv, EnvBasicTestWithParam,
-                        ::testing::Values(mem_env.get()));
+                        ::testing::Values(&GetMemoryEnv));
 
 namespace {
 
@@ -74,31 +127,15 @@ namespace {
 //
 // The purpose of returning an empty vector (instead of nullptr) is that gtest
 // ValuesIn() will skip running tests when given an empty collection.
-std::vector<Env*> GetCustomEnvs() {
-  static bool init = false;
-  static std::vector<Env*> res;
-  if (!init) {
-    init = true;
+std::vector<CreateEnvFunc*> GetCustomEnvs() {
+  std::vector<CreateEnvFunc*> res;
   const char* uri = getenv("TEST_ENV_URI");
   if (uri != nullptr) {
-      static std::shared_ptr<Env> env_guard;
-      static Env* custom_env;
-      Status s =
-          Env::CreateFromUri(ConfigOptions(), uri, "", &custom_env, &env_guard);
-      if (s.ok()) {
-        res.emplace_back(custom_env);
-      }
+    res.push_back(&GetTestEnv);
   }
   uri = getenv("TEST_FS_URI");
   if (uri != nullptr) {
-      static std::shared_ptr<Env> fs_env_guard;
-      static Env* fs_env;
-      Status s =
-          Env::CreateFromUri(ConfigOptions(), "", uri, &fs_env, &fs_env_guard);
-      if (s.ok()) {
-        res.emplace_back(fs_env);
-      }
-    }
+    res.push_back(&GetTestFS);
   }
   return res;
 }
@@ -110,7 +147,6 @@ INSTANTIATE_TEST_CASE_P(CustomEnv, EnvBasicTestWithParam,
 
 INSTANTIATE_TEST_CASE_P(CustomEnv, EnvMoreTestWithParam,
                         ::testing::ValuesIn(GetCustomEnvs()));
-
 #endif  // ROCKSDB_LITE
 
 TEST_P(EnvBasicTestWithParam, Basics) {

env/env_encryption.cc

@@ -18,62 +18,16 @@
 #include "rocksdb/convenience.h"
 #include "rocksdb/io_status.h"
 #include "rocksdb/system_clock.h"
+#include "rocksdb/utilities/customizable_util.h"
+#include "rocksdb/utilities/options_type.h"
 #include "util/aligned_buffer.h"
 #include "util/coding.h"
 #include "util/random.h"
 #include "util/string_util.h"
 
 #endif
-
 namespace ROCKSDB_NAMESPACE {
-
 #ifndef ROCKSDB_LITE
-static constexpr char kROT13CipherName[] = "ROT13";
-static constexpr char kCTRProviderName[] = "CTR";
-
-Status BlockCipher::CreateFromString(const ConfigOptions& /*config_options*/,
-                                     const std::string& value,
-                                     std::shared_ptr<BlockCipher>* result) {
-  std::string id = value;
-  size_t colon = value.find(':');
-  if (colon != std::string::npos) {
-    id = value.substr(0, colon);
-  }
-  if (id == kROT13CipherName) {
-    if (colon != std::string::npos) {
-      size_t block_size = ParseSizeT(value.substr(colon + 1));
-      result->reset(new ROT13BlockCipher(block_size));
-    } else {
-      result->reset(new ROT13BlockCipher(32));
-    }
-    return Status::OK();
-  } else {
-    return Status::NotSupported("Could not find cipher ", value);
-  }
-}
-
-Status EncryptionProvider::CreateFromString(
-    const ConfigOptions& /*config_options*/, const std::string& value,
-    std::shared_ptr<EncryptionProvider>* result) {
-  std::string id = value;
-  bool is_test = StartsWith(value, "test://");
-  Status status = Status::OK();
-  if (is_test) {
-    id = value.substr(strlen("test://"));
-  }
-  if (id == kCTRProviderName) {
-    result->reset(new CTREncryptionProvider());
-  } else if (is_test) {
-    result->reset(new CTREncryptionProvider());
-  } else {
-    return Status::NotSupported("Could not find provider ", value);
-  }
-  if (status.ok() && is_test) {
-    status = result->get()->TEST_Initialize();
-  }
-  return status;
-}
-
 std::shared_ptr<EncryptionProvider> EncryptionProvider::NewCTRProvider(
     const std::shared_ptr<BlockCipher>& cipher) {
   return std::make_shared<CTREncryptionProvider>(cipher);
@@ -1061,20 +1015,53 @@ Status BlockAccessCipherStream::Decrypt(uint64_t fileOffset, char *data, size_t
   }
 }
 
-const char* ROT13BlockCipher::Name() const { return kROT13CipherName; }
+namespace {
+static std::unordered_map<std::string, OptionTypeInfo>
+    rot13_block_cipher_type_info = {
+        {"block_size",
+         {0 /* No offset, whole struct*/, OptionType::kInt,
+          OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+};
+// Implements a BlockCipher using ROT13.
+//
+// Note: This is a sample implementation of BlockCipher,
+// it is NOT considered safe and should NOT be used in production.
+class ROT13BlockCipher : public BlockCipher {
+ private:
+  size_t blockSize_;
+
+ public:
+  explicit ROT13BlockCipher(size_t blockSize) : blockSize_(blockSize) {
+    RegisterOptions("ROT13BlockCipherOptions", &blockSize_,
+                    &rot13_block_cipher_type_info);
+  }
+
+  static const char* kClassName() { return "ROT13"; }
+  const char* Name() const override { return kClassName(); }
+  // BlockSize returns the size of each block supported by this cipher stream.
+  size_t BlockSize() override { return blockSize_; }
 
-// Encrypt a block of data.
-// Length of data is equal to BlockSize().
-Status ROT13BlockCipher::Encrypt(char* data) {
+  // Encrypt a block of data.
+  // Length of data is equal to BlockSize().
+  Status Encrypt(char* data) override {
     for (size_t i = 0; i < blockSize_; ++i) {
       data[i] += 13;
     }
     return Status::OK();
-}
+  }
 
-// Decrypt a block of data.
-// Length of data is equal to BlockSize().
-Status ROT13BlockCipher::Decrypt(char* data) { return Encrypt(data); }
+  // Decrypt a block of data.
+  // Length of data is equal to BlockSize().
+  Status Decrypt(char* data) override { return Encrypt(data); }
+};
+static const std::unordered_map<std::string, OptionTypeInfo>
+    ctr_encryption_provider_type_info = {
+        {"cipher",
+         OptionTypeInfo::AsCustomSharedPtr<BlockCipher>(
+             0 /* No offset, whole struct*/, OptionVerificationType::kByName,
+             OptionTypeFlags::kNone)},
+};
+}  // anonymous namespace
 
 // Allocate scratch space which is passed to EncryptBlock/DecryptBlock.
 void CTRCipherStream::AllocateScratch(std::string& scratch) {
@@ -1112,7 +1099,11 @@ Status CTRCipherStream::DecryptBlock(uint64_t blockIndex, char* data,
   return EncryptBlock(blockIndex, data, scratch);
 }
 
-const char* CTREncryptionProvider::Name() const { return kCTRProviderName; }
+CTREncryptionProvider::CTREncryptionProvider(
+    const std::shared_ptr<BlockCipher>& c)
+    : cipher_(c) {
+  RegisterOptions("Cipher", &cipher_, &ctr_encryption_provider_type_info);
+}
 
 // GetPrefixLength returns the length of the prefix that is added to every file
 // and used for storing encryption options.
@@ -1122,20 +1113,12 @@ size_t CTREncryptionProvider::GetPrefixLength() const {
   return defaultPrefixLength;
 }
 
-Status CTREncryptionProvider::TEST_Initialize() {
-  if (!cipher_) {
-    return BlockCipher::CreateFromString(
-        ConfigOptions(), std::string(kROT13CipherName) + ":32", &cipher_);
-  }
-  return Status::OK();
-}
-
 Status CTREncryptionProvider::AddCipher(const std::string& /*descriptor*/,
                                         const char* cipher, size_t len,
                                         bool /*for_write*/) {
   if (cipher_) {
     return Status::NotSupported("Cannot add keys to CTREncryptionProvider");
-  } else if (strcmp(kROT13CipherName, cipher) == 0) {
+  } else if (strcmp(ROT13BlockCipher::kClassName(), cipher) == 0) {
     cipher_.reset(new ROT13BlockCipher(len));
     return Status::OK();
   } else {
@@ -1252,6 +1235,70 @@ Status CTREncryptionProvider::CreateCipherStreamFromPrefix(
   return Status::OK();
 }
 
+namespace {
+static void RegisterEncryptionBuiltins() {
+  static std::once_flag once;
+  std::call_once(once, [&]() {
+    auto lib = ObjectRegistry::Default()->AddLibrary("encryption");
+    std::string ctr =
+        std::string(CTREncryptionProvider::kClassName()) + "?(://test)";
+    lib->Register<EncryptionProvider>(
+        std::string(CTREncryptionProvider::kClassName()) + "(://test)?",
+        [](const std::string& uri, std::unique_ptr<EncryptionProvider>* guard,
+           std::string* /*errmsg*/) {
+          if (EndsWith(uri, "://test")) {
+            std::shared_ptr<BlockCipher> cipher =
+                std::make_shared<ROT13BlockCipher>(32);
+            guard->reset(new CTREncryptionProvider(cipher));
+          } else {
+            guard->reset(new CTREncryptionProvider());
+          }
+          return guard->get();
+        });
+
+    lib->Register<EncryptionProvider>(
+        "1://test", [](const std::string& /*uri*/,
+                       std::unique_ptr<EncryptionProvider>* guard,
+                       std::string* /*errmsg*/) {
+          std::shared_ptr<BlockCipher> cipher =
+              std::make_shared<ROT13BlockCipher>(32);
+          guard->reset(new CTREncryptionProvider(cipher));
+          return guard->get();
+        });
+
+    lib->Register<BlockCipher>(
+        std::string(ROT13BlockCipher::kClassName()) + "(:.*)?",
+        [](const std::string& uri, std::unique_ptr<BlockCipher>* guard,
+           std::string* /* errmsg */) {
+          size_t colon = uri.find(':');
+          if (colon != std::string::npos) {
+            size_t block_size = ParseSizeT(uri.substr(colon + 1));
+            guard->reset(new ROT13BlockCipher(block_size));
+          } else {
+            guard->reset(new ROT13BlockCipher(32));
+          }
+
+          return guard->get();
+        });
+  });
+}
+}  // namespace
+
+Status BlockCipher::CreateFromString(const ConfigOptions& config_options,
+                                     const std::string& value,
+                                     std::shared_ptr<BlockCipher>* result) {
+  RegisterEncryptionBuiltins();
+  return LoadSharedObject<BlockCipher>(config_options, value, nullptr, result);
+}
+
+Status EncryptionProvider::CreateFromString(
+    const ConfigOptions& config_options, const std::string& value,
+    std::shared_ptr<EncryptionProvider>* result) {
+  RegisterEncryptionBuiltins();
+  return LoadSharedObject<EncryptionProvider>(config_options, value, nullptr,
+                                              result);
+}
+
 #endif // ROCKSDB_LITE
 
 }  // namespace ROCKSDB_NAMESPACE

env/env_encryption_ctr.h

@@ -10,31 +10,6 @@
 #include "rocksdb/env_encryption.h"
 
 namespace ROCKSDB_NAMESPACE {
-
-// Implements a BlockCipher using ROT13.
-//
-// Note: This is a sample implementation of BlockCipher,
-// it is NOT considered safe and should NOT be used in production.
-class ROT13BlockCipher : public BlockCipher {
- private:
-  size_t blockSize_;
-
- public:
-  ROT13BlockCipher(size_t blockSize) : blockSize_(blockSize) {}
-  virtual ~ROT13BlockCipher(){};
-  const char* Name() const override;
-  // BlockSize returns the size of each block supported by this cipher stream.
-  size_t BlockSize() override { return blockSize_; }
-
-  // Encrypt a block of data.
-  // Length of data is equal to BlockSize().
-  Status Encrypt(char* data) override;
-
-  // Decrypt a block of data.
-  // Length of data is equal to BlockSize().
-  Status Decrypt(char* data) override;
-};
-
 // CTRCipherStream implements BlockAccessCipherStream using an
 // Counter operations mode.
 // See https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation
@@ -86,11 +61,11 @@ class CTREncryptionProvider : public EncryptionProvider {
 
  public:
   explicit CTREncryptionProvider(
-      const std::shared_ptr<BlockCipher>& c = nullptr)
-      : cipher_(c){};
+      const std::shared_ptr<BlockCipher>& c = nullptr);
   virtual ~CTREncryptionProvider() {}
 
-  const char* Name() const override;
+  static const char* kClassName() { return "CTR"; }
+  const char* Name() const override { return kClassName(); }
 
   // GetPrefixLength returns the length of the prefix that is added to every
   // file
@@ -112,9 +87,7 @@ class CTREncryptionProvider : public EncryptionProvider {
 
   Status AddCipher(const std::string& descriptor, const char* /*cipher*/,
                    size_t /*len*/, bool /*for_write*/) override;
-
  protected:
-  Status TEST_Initialize() override;
 
   // PopulateSecretPrefixPart initializes the data into a new prefix block
   // that will be encrypted. This function will store the data in plain text.

env/env_test.cc

@@ -36,10 +36,13 @@
 #endif
 
 #include "env/env_chroot.h"
+#include "env/env_encryption_ctr.h"
 #include "logging/log_buffer.h"
 #include "port/malloc.h"
 #include "port/port.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/env.h"
+#include "rocksdb/env_encryption.h"
 #include "rocksdb/system_clock.h"
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
@@ -2387,6 +2390,74 @@ TEST_F(EnvTest, EnvWriteVerificationTest) {
   ASSERT_OK(s);
 }
 
+#ifndef ROCKSDB_LITE
+class EncryptionProviderTest : public testing::Test {
+ public:
+};
+
+TEST_F(EncryptionProviderTest, LoadCTRProvider) {
+  ConfigOptions config_options;
+  config_options.invoke_prepare_options = false;
+  std::string CTR = CTREncryptionProvider::kClassName();
+  std::shared_ptr<EncryptionProvider> provider;
+  // Test a provider with no cipher
+  ASSERT_OK(
+      EncryptionProvider::CreateFromString(config_options, CTR, &provider));
+  ASSERT_NE(provider, nullptr);
+  ASSERT_EQ(provider->Name(), CTR);
+  ASSERT_NOK(provider->PrepareOptions(config_options));
+  ASSERT_NOK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
+  auto cipher = provider->GetOptions<std::shared_ptr<BlockCipher>>("Cipher");
+  ASSERT_NE(cipher, nullptr);
+  ASSERT_EQ(cipher->get(), nullptr);
+  provider.reset();
+
+  ASSERT_OK(EncryptionProvider::CreateFromString(config_options,
+                                                 CTR + "://test", &provider));
+  ASSERT_NE(provider, nullptr);
+  ASSERT_EQ(provider->Name(), CTR);
+  ASSERT_OK(provider->PrepareOptions(config_options));
+  ASSERT_OK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
+  cipher = provider->GetOptions<std::shared_ptr<BlockCipher>>("Cipher");
+  ASSERT_NE(cipher, nullptr);
+  ASSERT_NE(cipher->get(), nullptr);
+  ASSERT_STREQ(cipher->get()->Name(), "ROT13");
+  provider.reset();
+
+  ASSERT_OK(EncryptionProvider::CreateFromString(config_options, "1://test",
+                                                 &provider));
+  ASSERT_NE(provider, nullptr);
+  ASSERT_EQ(provider->Name(), CTR);
+  ASSERT_OK(provider->PrepareOptions(config_options));
+  ASSERT_OK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
+  cipher = provider->GetOptions<std::shared_ptr<BlockCipher>>("Cipher");
+  ASSERT_NE(cipher, nullptr);
+  ASSERT_NE(cipher->get(), nullptr);
+  ASSERT_STREQ(cipher->get()->Name(), "ROT13");
+  provider.reset();
+
+  ASSERT_OK(EncryptionProvider::CreateFromString(
+      config_options, "id=" + CTR + "; cipher=ROT13", &provider));
+  ASSERT_NE(provider, nullptr);
+  ASSERT_EQ(provider->Name(), CTR);
+  cipher = provider->GetOptions<std::shared_ptr<BlockCipher>>("Cipher");
+  ASSERT_NE(cipher, nullptr);
+  ASSERT_NE(cipher->get(), nullptr);
+  ASSERT_STREQ(cipher->get()->Name(), "ROT13");
+  provider.reset();
+}
+
+TEST_F(EncryptionProviderTest, LoadROT13Cipher) {
+  ConfigOptions config_options;
+  std::shared_ptr<BlockCipher> cipher;
+  // Test a provider with no cipher
+  ASSERT_OK(BlockCipher::CreateFromString(config_options, "ROT13", &cipher));
+  ASSERT_NE(cipher, nullptr);
+  ASSERT_STREQ(cipher->Name(), "ROT13");
+}
+
+#endif  // ROCKSDB_LITE
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

include/rocksdb/env_encryption.h

@@ -9,6 +9,7 @@
 
 #include <string>
 
+#include "rocksdb/customizable.h"
 #include "rocksdb/env.h"
 #include "rocksdb/file_system.h"
 #include "rocksdb/rocksdb_namespace.h"
@@ -58,7 +59,7 @@ class BlockAccessCipherStream {
 };
 
 // BlockCipher
-class BlockCipher {
+class BlockCipher : public Customizable {
  public:
   virtual ~BlockCipher(){};
 
@@ -80,12 +81,12 @@ class BlockCipher {
                                  const std::string& value,
                                  std::shared_ptr<BlockCipher>* result);
 
+  static const char* Type() { return "BlockCipher"; }
   // Short-cut method to create a ROT13 BlockCipher.
   // This cipher is only suitable for test purposes and should not be used in
   // production!!!
   static std::shared_ptr<BlockCipher> NewROT13Cipher(size_t block_size);
 
-  virtual const char* Name() const = 0;
   // BlockSize returns the size of each block supported by this cipher stream.
   virtual size_t BlockSize() = 0;
 
@@ -101,7 +102,7 @@ class BlockCipher {
 // The encryption provider is used to create a cipher stream for a specific
 // file. The returned cipher stream will be used for actual
 // encryption/decryption actions.
-class EncryptionProvider {
+class EncryptionProvider : public Customizable {
  public:
   virtual ~EncryptionProvider(){};
 
@@ -109,14 +110,14 @@ class EncryptionProvider {
   // The value describes the type of provider (and potentially optional
   // configuration parameters) used to create this provider.
   // For example, if the value is "CTR", a CTREncryptionProvider will be
-  // created. If the value is preceded by "test://" (e.g test://CTR"), the
-  // TEST_Initialize method will be invoked prior to returning the provider.
+  // created. If the value is ends with "://test" (e.g CTR://test"), the
+  // provider will be initialized in "TEST" mode prior to being returned.
   //
   // @param config_options  Options to control how this provider is created
   //                        and initialized.
   // @param value  The value might be:
   //   - CTR         Create a CTR provider
-  //   - test://CTR Create a CTR provider and initialize it for tests.
+  //   - CTR://test Create a CTR provider and initialize it for tests.
   // @param result The new provider object
   // @return OK if the provider was successfully created
   // @return NotFound if an invalid name was specified in the value
@@ -125,13 +126,12 @@ class EncryptionProvider {
                                  const std::string& value,
                                  std::shared_ptr<EncryptionProvider>* result);
 
+  static const char* Type() { return "EncryptionProvider"; }
+
   // Short-cut method to create a CTR-provider
   static std::shared_ptr<EncryptionProvider> NewCTRProvider(
       const std::shared_ptr<BlockCipher>& cipher);
 
-  // Returns the name of this EncryptionProvider
-  virtual const char* Name() const = 0;
-
   // GetPrefixLength returns the length of the prefix that is added to every
   // file and used for storing encryption options. For optimal performance, the
   // prefix length should be a multiple of the page size.
@@ -165,11 +165,6 @@ class EncryptionProvider {
   // or not a file is encrypted by this provider.  The maker will also be part
   // of any encryption prefix for this provider.
   virtual std::string GetMarker() const { return ""; }
-
- protected:
-  // Optional method to initialize an EncryptionProvider in the TEST
-  // environment.
-  virtual Status TEST_Initialize() { return Status::OK(); }
 };
 
 class EncryptedSequentialFile : public FSSequentialFile {

options/configurable.cc

@@ -40,8 +40,9 @@ void Configurable::RegisterOptions(
 //*************************************************************************
 
 Status Configurable::PrepareOptions(const ConfigOptions& opts) {
+  // We ignore the invoke_prepare_options here intentionally,
+  // as if you are here, you must have called PrepareOptions explicitly.
   Status status = Status::OK();
-  if (opts.invoke_prepare_options) {
 #ifndef ROCKSDB_LITE
   for (auto opt_iter : options_) {
     for (auto map_iter : *(opt_iter.type_map)) {
@@ -57,18 +58,19 @@ Status Configurable::PrepareOptions(const ConfigOptions& opts) {
               return status;
             }
           } else if (!opt_info.CanBeNull()) {
-              status = Status::NotFound("Missing configurable object",
-                                        map_iter.first);
+            status =
+                Status::NotFound("Missing configurable object", map_iter.first);
           }
         }
       }
     }
   }
+#else
+  (void)opts;
 #endif  // ROCKSDB_LITE
   if (status.ok()) {
     is_prepared_ = true;
   }
-  }
   return status;
 }
 

options/customizable_test.cc

@@ -18,6 +18,7 @@
 #include "options/options_helper.h"
 #include "options/options_parser.h"
 #include "rocksdb/convenience.h"
+#include "rocksdb/env_encryption.h"
 #include "rocksdb/flush_block_policy.h"
 #include "rocksdb/secondary_cache.h"
 #include "rocksdb/utilities/customizable_util.h"
@@ -27,6 +28,7 @@
 #include "table/mock_table.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
+#include "util/string_util.h"
 
 #ifndef GFLAGS
 bool FLAGS_enable_print = false;
@@ -231,15 +233,18 @@ static std::unordered_map<std::string, OptionTypeInfo> simple_option_info = {
     {"bool",
      {offsetof(struct SimpleOptions, b), OptionType::kBoolean,
       OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
-    {"unique", OptionTypeInfo::AsCustomUniquePtr<TestCustomizable>(
-                   offsetof(struct SimpleOptions, cu),
-                   OptionVerificationType::kNormal, OptionTypeFlags::kNone)},
-    {"shared", OptionTypeInfo::AsCustomSharedPtr<TestCustomizable>(
-                   offsetof(struct SimpleOptions, cs),
-                   OptionVerificationType::kNormal, OptionTypeFlags::kNone)},
-    {"pointer", OptionTypeInfo::AsCustomRawPtr<TestCustomizable>(
-                    offsetof(struct SimpleOptions, cp),
-                    OptionVerificationType::kNormal, OptionTypeFlags::kNone)},
+    {"unique",
+     OptionTypeInfo::AsCustomUniquePtr<TestCustomizable>(
+         offsetof(struct SimpleOptions, cu), OptionVerificationType::kNormal,
+         OptionTypeFlags::kAllowNull)},
+    {"shared",
+     OptionTypeInfo::AsCustomSharedPtr<TestCustomizable>(
+         offsetof(struct SimpleOptions, cs), OptionVerificationType::kNormal,
+         OptionTypeFlags::kAllowNull)},
+    {"pointer",
+     OptionTypeInfo::AsCustomRawPtr<TestCustomizable>(
+         offsetof(struct SimpleOptions, cp), OptionVerificationType::kNormal,
+         OptionTypeFlags::kAllowNull)},
 #endif  // ROCKSDB_LITE
 };
 
@@ -554,11 +559,6 @@ TEST_F(CustomizableTest, PrepareOptionsTest) {
   ASSERT_FALSE(simple->cp->IsPrepared());
 
   ASSERT_OK(base->PrepareOptions(config_options_));
-  ASSERT_FALSE(base->IsPrepared());
-  ASSERT_FALSE(simple->cu->IsPrepared());
-  ASSERT_FALSE(simple->cs->IsPrepared());
-  ASSERT_FALSE(simple->cp->IsPrepared());
-  ASSERT_OK(base->PrepareOptions(prepared));
   ASSERT_TRUE(base->IsPrepared());
   ASSERT_TRUE(simple->cu->IsPrepared());
   ASSERT_TRUE(simple->cs->IsPrepared());
@@ -917,6 +917,48 @@ static int RegisterTestObjects(ObjectLibrary& library,
   return static_cast<int>(library.GetFactoryCount(&num_types));
 }
 
+class MockEncryptionProvider : public EncryptionProvider {
+ public:
+  explicit MockEncryptionProvider(const std::string& id) : id_(id) {}
+  const char* Name() const override { return "Mock"; }
+  size_t GetPrefixLength() const override { return 0; }
+  Status CreateNewPrefix(const std::string& /*fname*/, char* /*prefix*/,
+                         size_t /*prefixLength*/) const override {
+    return Status::NotSupported();
+  }
+
+  Status AddCipher(const std::string& /*descriptor*/, const char* /*cipher*/,
+                   size_t /*len*/, bool /*for_write*/) override {
+    return Status::NotSupported();
+  }
+
+  Status CreateCipherStream(
+      const std::string& /*fname*/, const EnvOptions& /*options*/,
+      Slice& /*prefix*/,
+      std::unique_ptr<BlockAccessCipherStream>* /*result*/) override {
+    return Status::NotSupported();
+  }
+  Status ValidateOptions(const DBOptions& db_opts,
+                         const ColumnFamilyOptions& cf_opts) const override {
+    if (EndsWith(id_, "://test")) {
+      return EncryptionProvider::ValidateOptions(db_opts, cf_opts);
+    } else {
+      return Status::InvalidArgument("MockProvider not initialized");
+    }
+  }
+
+ private:
+  std::string id_;
+};
+
+class MockCipher : public BlockCipher {
+ public:
+  const char* Name() const override { return "Mock"; }
+  size_t BlockSize() override { return 0; }
+  Status Encrypt(char* /*data*/) override { return Status::NotSupported(); }
+  Status Decrypt(char* data) override { return Encrypt(data); }
+};
+
 class TestFlushBlockPolicyFactory : public FlushBlockPolicyFactory {
  public:
   TestFlushBlockPolicyFactory() {}
@@ -935,6 +977,19 @@ static int RegisterLocalObjects(ObjectLibrary& library,
                                 const std::string& /*arg*/) {
   size_t num_types;
   // Load any locally defined objects here
+  library.Register<EncryptionProvider>(
+      "Mock(://test)?",
+      [](const std::string& uri, std::unique_ptr<EncryptionProvider>* guard,
+         std::string* /* errmsg */) {
+        guard->reset(new MockEncryptionProvider(uri));
+        return guard->get();
+      });
+  library.Register<BlockCipher>("Mock", [](const std::string& /*uri*/,
+                                           std::unique_ptr<BlockCipher>* guard,
+                                           std::string* /* errmsg */) {
+    guard->reset(new MockCipher());
+    return guard->get();
+  });
   library.Register<FlushBlockPolicyFactory>(
       TestFlushBlockPolicyFactory::kClassName(),
       [](const std::string& /*uri*/,
@@ -957,7 +1012,10 @@ static int RegisterLocalObjects(ObjectLibrary& library,
 
 class LoadCustomizableTest : public testing::Test {
  public:
-  LoadCustomizableTest() { config_options_.ignore_unsupported_options = false; }
+  LoadCustomizableTest() {
+    config_options_.ignore_unsupported_options = false;
+    config_options_.invoke_prepare_options = false;
+  }
   bool RegisterTests(const std::string& arg) {
 #ifndef ROCKSDB_LITE
     config_options_.registry->AddLibrary("custom-tests", RegisterTestObjects,
@@ -1047,6 +1105,49 @@ TEST_F(LoadCustomizableTest, LoadComparatorTest) {
   }
 }
 
+#ifndef ROCKSDB_LITE
+TEST_F(LoadCustomizableTest, LoadEncryptionProviderTest) {
+  std::shared_ptr<EncryptionProvider> result;
+  ASSERT_NOK(
+      EncryptionProvider::CreateFromString(config_options_, "Mock", &result));
+  ASSERT_OK(
+      EncryptionProvider::CreateFromString(config_options_, "CTR", &result));
+  ASSERT_NE(result, nullptr);
+  ASSERT_STREQ(result->Name(), "CTR");
+  ASSERT_NOK(result->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
+  ASSERT_OK(EncryptionProvider::CreateFromString(config_options_, "CTR://test",
+                                                 &result));
+  ASSERT_NE(result, nullptr);
+  ASSERT_STREQ(result->Name(), "CTR");
+  ASSERT_OK(result->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
+
+  if (RegisterTests("Test")) {
+    ASSERT_OK(
+        EncryptionProvider::CreateFromString(config_options_, "Mock", &result));
+    ASSERT_NE(result, nullptr);
+    ASSERT_STREQ(result->Name(), "Mock");
+    ASSERT_OK(EncryptionProvider::CreateFromString(config_options_,
+                                                   "Mock://test", &result));
+    ASSERT_NE(result, nullptr);
+    ASSERT_STREQ(result->Name(), "Mock");
+    ASSERT_OK(result->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
+  }
+}
+
+TEST_F(LoadCustomizableTest, LoadEncryptionCipherTest) {
+  std::shared_ptr<BlockCipher> result;
+  ASSERT_NOK(BlockCipher::CreateFromString(config_options_, "Mock", &result));
+  ASSERT_OK(BlockCipher::CreateFromString(config_options_, "ROT13", &result));
+  ASSERT_NE(result, nullptr);
+  ASSERT_STREQ(result->Name(), "ROT13");
+  if (RegisterTests("Test")) {
+    ASSERT_OK(BlockCipher::CreateFromString(config_options_, "Mock", &result));
+    ASSERT_NE(result, nullptr);
+    ASSERT_STREQ(result->Name(), "Mock");
+  }
+}
+#endif  // !ROCKSDB_LITE
+
 TEST_F(LoadCustomizableTest, LoadFlushBlockPolicyFactoryTest) {
   std::shared_ptr<TableFactory> table;
   std::shared_ptr<FlushBlockPolicyFactory> result;