Make EncryptionProvider and BlockCipher into Customizable objects (#8354)

Summary:
Made the EncryptionProvider and BlockCipher classes inherit from Customizable.  Added/fixed the CreateFromString method to these classes to create instances from builtin or registered classes.  Added tests to verify that instances can be registered and retrieved as appropriate.

Added the ability to configure the builtin (CTR, ROT13) classes from configurable properties.  Added the appropriate tests.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8354

Reviewed By: zhichao-cao

Differential Revision: D29558949

Pulled By: mrambacher

fbshipit-source-id: c20286b32d179777e060f51a58943e9b0cf81d04
main
Mark Rambacher 3 years ago committed by Facebook GitHub Bot
parent aeb913dd01
commit 42ba60b3ba
  1. 10
      db/db_test_util.cc
  2. 122
      env/env_basic_test.cc
  3. 179
      env/env_encryption.cc
  4. 33
      env/env_encryption_ctr.h
  5. 71
      env/env_test.cc
  6. 23
      include/rocksdb/env_encryption.h
  7. 10
      options/configurable.cc
  8. 131
      options/customizable_test.cc

@ -68,9 +68,13 @@ DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
if (getenv("ENCRYPTED_ENV")) { if (getenv("ENCRYPTED_ENV")) {
std::shared_ptr<EncryptionProvider> provider; std::shared_ptr<EncryptionProvider> provider;
Status s = EncryptionProvider::CreateFromString( std::string provider_id = getenv("ENCRYPTED_ENV");
config_options, std::string("test://") + getenv("ENCRYPTED_ENV"), if (provider_id.find("=") == std::string::npos &&
&provider); !EndsWith(provider_id, "://test")) {
provider_id = provider_id + "://test";
}
EXPECT_OK(EncryptionProvider::CreateFromString(ConfigOptions(), provider_id,
&provider));
encrypted_env_ = NewEncryptedEnv(mem_env_ ? mem_env_ : base_env, provider); encrypted_env_ = NewEncryptedEnv(mem_env_ ? mem_env_ : base_env, provider);
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE

@ -17,15 +17,81 @@
#include "test_util/testharness.h" #include "test_util/testharness.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
typedef Env* CreateEnvFunc();
namespace {
// These functions are used to create the various environments under which this
// test can execute. These functions are used to allow the test cases to be
// created without the Env being initialized, thereby eliminating a potential
// static initialization fiasco/race condition when attempting to get a
// custom/configured env prior to main being invoked.
static Env* GetDefaultEnv() { return Env::Default(); }
static Env* GetMockEnv() {
static std::unique_ptr<Env> mock_env(new MockEnv(Env::Default()));
return mock_env.get();
}
#ifndef ROCKSDB_LITE
static Env* NewTestEncryptedEnv(Env* base, const std::string& provider_id) {
ConfigOptions config_opts;
config_opts.invoke_prepare_options = false;
std::shared_ptr<EncryptionProvider> provider;
EXPECT_OK(EncryptionProvider::CreateFromString(config_opts, provider_id,
&provider));
return NewEncryptedEnv(base, provider);
}
static Env* GetCtrEncryptedEnv() {
static std::unique_ptr<Env> ctr_encrypt_env(
NewTestEncryptedEnv(Env::Default(), "CTR://test"));
return ctr_encrypt_env.get();
}
static Env* GetMemoryEnv() {
static std::unique_ptr<Env> mem_env(NewMemEnv(Env::Default()));
return mem_env.get();
}
static Env* GetTestEnv() {
static std::shared_ptr<Env> env_guard;
static Env* custom_env = nullptr;
if (custom_env == nullptr) {
const char* uri = getenv("TEST_ENV_URI");
if (uri != nullptr) {
EXPECT_OK(Env::CreateFromUri(ConfigOptions(), uri, "", &custom_env,
&env_guard));
}
}
EXPECT_NE(custom_env, nullptr);
return custom_env;
}
class EnvBasicTestWithParam : public testing::Test, static Env* GetTestFS() {
public ::testing::WithParamInterface<Env*> { static std::shared_ptr<Env> fs_env_guard;
static Env* fs_env = nullptr;
if (fs_env == nullptr) {
const char* uri = getenv("TEST_FS_URI");
if (uri != nullptr) {
EXPECT_OK(
Env::CreateFromUri(ConfigOptions(), uri, "", &fs_env, &fs_env_guard));
}
}
EXPECT_NE(fs_env, nullptr);
return fs_env;
}
#endif // ROCKSDB_LITE
} // namespace
class EnvBasicTestWithParam
: public testing::Test,
public ::testing::WithParamInterface<CreateEnvFunc*> {
public: public:
Env* env_; Env* env_;
const EnvOptions soptions_; const EnvOptions soptions_;
std::string test_dir_; std::string test_dir_;
EnvBasicTestWithParam() : env_(GetParam()) { EnvBasicTestWithParam() : env_(GetParam()()) {
test_dir_ = test::PerThreadDBPath(env_, "env_basic_test"); test_dir_ = test::PerThreadDBPath(env_, "env_basic_test");
} }
@ -37,35 +103,22 @@ class EnvBasicTestWithParam : public testing::Test,
class EnvMoreTestWithParam : public EnvBasicTestWithParam {}; class EnvMoreTestWithParam : public EnvBasicTestWithParam {};
INSTANTIATE_TEST_CASE_P(EnvDefault, EnvBasicTestWithParam, INSTANTIATE_TEST_CASE_P(EnvDefault, EnvBasicTestWithParam,
::testing::Values(Env::Default())); ::testing::Values(&GetDefaultEnv));
INSTANTIATE_TEST_CASE_P(EnvDefault, EnvMoreTestWithParam, INSTANTIATE_TEST_CASE_P(EnvDefault, EnvMoreTestWithParam,
::testing::Values(Env::Default())); ::testing::Values(&GetDefaultEnv));
static std::unique_ptr<Env> mock_env(new MockEnv(Env::Default()));
INSTANTIATE_TEST_CASE_P(MockEnv, EnvBasicTestWithParam, INSTANTIATE_TEST_CASE_P(MockEnv, EnvBasicTestWithParam,
::testing::Values(mock_env.get())); ::testing::Values(&GetMockEnv));
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
static Env* NewTestEncryptedEnv(Env* base, const std::string& provider_id) {
std::shared_ptr<EncryptionProvider> provider;
EXPECT_OK(EncryptionProvider::CreateFromString(ConfigOptions(), provider_id,
&provider));
return NewEncryptedEnv(base, provider);
}
// next statements run env test against default encryption code. // next statements run env test against default encryption code.
static std::unique_ptr<Env> ctr_encrypt_env(NewTestEncryptedEnv(Env::Default(),
"test://CTR"));
INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvBasicTestWithParam, INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvBasicTestWithParam,
::testing::Values(ctr_encrypt_env.get())); ::testing::Values(&GetCtrEncryptedEnv));
INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvMoreTestWithParam, INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvMoreTestWithParam,
::testing::Values(ctr_encrypt_env.get())); ::testing::Values(&GetCtrEncryptedEnv));
#endif // ROCKSDB_LITE
#ifndef ROCKSDB_LITE
static std::unique_ptr<Env> mem_env(NewMemEnv(Env::Default()));
INSTANTIATE_TEST_CASE_P(MemEnv, EnvBasicTestWithParam, INSTANTIATE_TEST_CASE_P(MemEnv, EnvBasicTestWithParam,
::testing::Values(mem_env.get())); ::testing::Values(&GetMemoryEnv));
namespace { namespace {
@ -74,31 +127,15 @@ namespace {
// //
// The purpose of returning an empty vector (instead of nullptr) is that gtest // The purpose of returning an empty vector (instead of nullptr) is that gtest
// ValuesIn() will skip running tests when given an empty collection. // ValuesIn() will skip running tests when given an empty collection.
std::vector<Env*> GetCustomEnvs() { std::vector<CreateEnvFunc*> GetCustomEnvs() {
static bool init = false; std::vector<CreateEnvFunc*> res;
static std::vector<Env*> res;
if (!init) {
init = true;
const char* uri = getenv("TEST_ENV_URI"); const char* uri = getenv("TEST_ENV_URI");
if (uri != nullptr) { if (uri != nullptr) {
static std::shared_ptr<Env> env_guard; res.push_back(&GetTestEnv);
static Env* custom_env;
Status s =
Env::CreateFromUri(ConfigOptions(), uri, "", &custom_env, &env_guard);
if (s.ok()) {
res.emplace_back(custom_env);
}
} }
uri = getenv("TEST_FS_URI"); uri = getenv("TEST_FS_URI");
if (uri != nullptr) { if (uri != nullptr) {
static std::shared_ptr<Env> fs_env_guard; res.push_back(&GetTestFS);
static Env* fs_env;
Status s =
Env::CreateFromUri(ConfigOptions(), "", uri, &fs_env, &fs_env_guard);
if (s.ok()) {
res.emplace_back(fs_env);
}
}
} }
return res; return res;
} }
@ -110,7 +147,6 @@ INSTANTIATE_TEST_CASE_P(CustomEnv, EnvBasicTestWithParam,
INSTANTIATE_TEST_CASE_P(CustomEnv, EnvMoreTestWithParam, INSTANTIATE_TEST_CASE_P(CustomEnv, EnvMoreTestWithParam,
::testing::ValuesIn(GetCustomEnvs())); ::testing::ValuesIn(GetCustomEnvs()));
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
TEST_P(EnvBasicTestWithParam, Basics) { TEST_P(EnvBasicTestWithParam, Basics) {

@ -18,62 +18,16 @@
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/io_status.h" #include "rocksdb/io_status.h"
#include "rocksdb/system_clock.h" #include "rocksdb/system_clock.h"
#include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/options_type.h"
#include "util/aligned_buffer.h" #include "util/aligned_buffer.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/random.h" #include "util/random.h"
#include "util/string_util.h" #include "util/string_util.h"
#endif #endif
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
static constexpr char kROT13CipherName[] = "ROT13";
static constexpr char kCTRProviderName[] = "CTR";
Status BlockCipher::CreateFromString(const ConfigOptions& /*config_options*/,
const std::string& value,
std::shared_ptr<BlockCipher>* result) {
std::string id = value;
size_t colon = value.find(':');
if (colon != std::string::npos) {
id = value.substr(0, colon);
}
if (id == kROT13CipherName) {
if (colon != std::string::npos) {
size_t block_size = ParseSizeT(value.substr(colon + 1));
result->reset(new ROT13BlockCipher(block_size));
} else {
result->reset(new ROT13BlockCipher(32));
}
return Status::OK();
} else {
return Status::NotSupported("Could not find cipher ", value);
}
}
Status EncryptionProvider::CreateFromString(
const ConfigOptions& /*config_options*/, const std::string& value,
std::shared_ptr<EncryptionProvider>* result) {
std::string id = value;
bool is_test = StartsWith(value, "test://");
Status status = Status::OK();
if (is_test) {
id = value.substr(strlen("test://"));
}
if (id == kCTRProviderName) {
result->reset(new CTREncryptionProvider());
} else if (is_test) {
result->reset(new CTREncryptionProvider());
} else {
return Status::NotSupported("Could not find provider ", value);
}
if (status.ok() && is_test) {
status = result->get()->TEST_Initialize();
}
return status;
}
std::shared_ptr<EncryptionProvider> EncryptionProvider::NewCTRProvider( std::shared_ptr<EncryptionProvider> EncryptionProvider::NewCTRProvider(
const std::shared_ptr<BlockCipher>& cipher) { const std::shared_ptr<BlockCipher>& cipher) {
return std::make_shared<CTREncryptionProvider>(cipher); return std::make_shared<CTREncryptionProvider>(cipher);
@ -1061,20 +1015,53 @@ Status BlockAccessCipherStream::Decrypt(uint64_t fileOffset, char *data, size_t
} }
} }
const char* ROT13BlockCipher::Name() const { return kROT13CipherName; } namespace {
static std::unordered_map<std::string, OptionTypeInfo>
rot13_block_cipher_type_info = {
{"block_size",
{0 /* No offset, whole struct*/, OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
};
// Implements a BlockCipher using ROT13.
//
// Note: This is a sample implementation of BlockCipher,
// it is NOT considered safe and should NOT be used in production.
class ROT13BlockCipher : public BlockCipher {
private:
size_t blockSize_;
public:
explicit ROT13BlockCipher(size_t blockSize) : blockSize_(blockSize) {
RegisterOptions("ROT13BlockCipherOptions", &blockSize_,
&rot13_block_cipher_type_info);
}
static const char* kClassName() { return "ROT13"; }
const char* Name() const override { return kClassName(); }
// BlockSize returns the size of each block supported by this cipher stream.
size_t BlockSize() override { return blockSize_; }
// Encrypt a block of data. // Encrypt a block of data.
// Length of data is equal to BlockSize(). // Length of data is equal to BlockSize().
Status ROT13BlockCipher::Encrypt(char* data) { Status Encrypt(char* data) override {
for (size_t i = 0; i < blockSize_; ++i) { for (size_t i = 0; i < blockSize_; ++i) {
data[i] += 13; data[i] += 13;
} }
return Status::OK(); return Status::OK();
} }
// Decrypt a block of data. // Decrypt a block of data.
// Length of data is equal to BlockSize(). // Length of data is equal to BlockSize().
Status ROT13BlockCipher::Decrypt(char* data) { return Encrypt(data); } Status Decrypt(char* data) override { return Encrypt(data); }
};
static const std::unordered_map<std::string, OptionTypeInfo>
ctr_encryption_provider_type_info = {
{"cipher",
OptionTypeInfo::AsCustomSharedPtr<BlockCipher>(
0 /* No offset, whole struct*/, OptionVerificationType::kByName,
OptionTypeFlags::kNone)},
};
} // anonymous namespace
// Allocate scratch space which is passed to EncryptBlock/DecryptBlock. // Allocate scratch space which is passed to EncryptBlock/DecryptBlock.
void CTRCipherStream::AllocateScratch(std::string& scratch) { void CTRCipherStream::AllocateScratch(std::string& scratch) {
@ -1112,7 +1099,11 @@ Status CTRCipherStream::DecryptBlock(uint64_t blockIndex, char* data,
return EncryptBlock(blockIndex, data, scratch); return EncryptBlock(blockIndex, data, scratch);
} }
const char* CTREncryptionProvider::Name() const { return kCTRProviderName; } CTREncryptionProvider::CTREncryptionProvider(
const std::shared_ptr<BlockCipher>& c)
: cipher_(c) {
RegisterOptions("Cipher", &cipher_, &ctr_encryption_provider_type_info);
}
// GetPrefixLength returns the length of the prefix that is added to every file // GetPrefixLength returns the length of the prefix that is added to every file
// and used for storing encryption options. // and used for storing encryption options.
@ -1122,20 +1113,12 @@ size_t CTREncryptionProvider::GetPrefixLength() const {
return defaultPrefixLength; return defaultPrefixLength;
} }
Status CTREncryptionProvider::TEST_Initialize() {
if (!cipher_) {
return BlockCipher::CreateFromString(
ConfigOptions(), std::string(kROT13CipherName) + ":32", &cipher_);
}
return Status::OK();
}
Status CTREncryptionProvider::AddCipher(const std::string& /*descriptor*/, Status CTREncryptionProvider::AddCipher(const std::string& /*descriptor*/,
const char* cipher, size_t len, const char* cipher, size_t len,
bool /*for_write*/) { bool /*for_write*/) {
if (cipher_) { if (cipher_) {
return Status::NotSupported("Cannot add keys to CTREncryptionProvider"); return Status::NotSupported("Cannot add keys to CTREncryptionProvider");
} else if (strcmp(kROT13CipherName, cipher) == 0) { } else if (strcmp(ROT13BlockCipher::kClassName(), cipher) == 0) {
cipher_.reset(new ROT13BlockCipher(len)); cipher_.reset(new ROT13BlockCipher(len));
return Status::OK(); return Status::OK();
} else { } else {
@ -1252,6 +1235,70 @@ Status CTREncryptionProvider::CreateCipherStreamFromPrefix(
return Status::OK(); return Status::OK();
} }
namespace {
static void RegisterEncryptionBuiltins() {
static std::once_flag once;
std::call_once(once, [&]() {
auto lib = ObjectRegistry::Default()->AddLibrary("encryption");
std::string ctr =
std::string(CTREncryptionProvider::kClassName()) + "?(://test)";
lib->Register<EncryptionProvider>(
std::string(CTREncryptionProvider::kClassName()) + "(://test)?",
[](const std::string& uri, std::unique_ptr<EncryptionProvider>* guard,
std::string* /*errmsg*/) {
if (EndsWith(uri, "://test")) {
std::shared_ptr<BlockCipher> cipher =
std::make_shared<ROT13BlockCipher>(32);
guard->reset(new CTREncryptionProvider(cipher));
} else {
guard->reset(new CTREncryptionProvider());
}
return guard->get();
});
lib->Register<EncryptionProvider>(
"1://test", [](const std::string& /*uri*/,
std::unique_ptr<EncryptionProvider>* guard,
std::string* /*errmsg*/) {
std::shared_ptr<BlockCipher> cipher =
std::make_shared<ROT13BlockCipher>(32);
guard->reset(new CTREncryptionProvider(cipher));
return guard->get();
});
lib->Register<BlockCipher>(
std::string(ROT13BlockCipher::kClassName()) + "(:.*)?",
[](const std::string& uri, std::unique_ptr<BlockCipher>* guard,
std::string* /* errmsg */) {
size_t colon = uri.find(':');
if (colon != std::string::npos) {
size_t block_size = ParseSizeT(uri.substr(colon + 1));
guard->reset(new ROT13BlockCipher(block_size));
} else {
guard->reset(new ROT13BlockCipher(32));
}
return guard->get();
});
});
}
} // namespace
Status BlockCipher::CreateFromString(const ConfigOptions& config_options,
const std::string& value,
std::shared_ptr<BlockCipher>* result) {
RegisterEncryptionBuiltins();
return LoadSharedObject<BlockCipher>(config_options, value, nullptr, result);
}
Status EncryptionProvider::CreateFromString(
const ConfigOptions& config_options, const std::string& value,
std::shared_ptr<EncryptionProvider>* result) {
RegisterEncryptionBuiltins();
return LoadSharedObject<EncryptionProvider>(config_options, value, nullptr,
result);
}
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -10,31 +10,6 @@
#include "rocksdb/env_encryption.h" #include "rocksdb/env_encryption.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// Implements a BlockCipher using ROT13.
//
// Note: This is a sample implementation of BlockCipher,
// it is NOT considered safe and should NOT be used in production.
class ROT13BlockCipher : public BlockCipher {
private:
size_t blockSize_;
public:
ROT13BlockCipher(size_t blockSize) : blockSize_(blockSize) {}
virtual ~ROT13BlockCipher(){};
const char* Name() const override;
// BlockSize returns the size of each block supported by this cipher stream.
size_t BlockSize() override { return blockSize_; }
// Encrypt a block of data.
// Length of data is equal to BlockSize().
Status Encrypt(char* data) override;
// Decrypt a block of data.
// Length of data is equal to BlockSize().
Status Decrypt(char* data) override;
};
// CTRCipherStream implements BlockAccessCipherStream using an // CTRCipherStream implements BlockAccessCipherStream using an
// Counter operations mode. // Counter operations mode.
// See https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation // See https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation
@ -86,11 +61,11 @@ class CTREncryptionProvider : public EncryptionProvider {
public: public:
explicit CTREncryptionProvider( explicit CTREncryptionProvider(
const std::shared_ptr<BlockCipher>& c = nullptr) const std::shared_ptr<BlockCipher>& c = nullptr);
: cipher_(c){};
virtual ~CTREncryptionProvider() {} virtual ~CTREncryptionProvider() {}
const char* Name() const override; static const char* kClassName() { return "CTR"; }
const char* Name() const override { return kClassName(); }
// GetPrefixLength returns the length of the prefix that is added to every // GetPrefixLength returns the length of the prefix that is added to every
// file // file
@ -112,9 +87,7 @@ class CTREncryptionProvider : public EncryptionProvider {
Status AddCipher(const std::string& descriptor, const char* /*cipher*/, Status AddCipher(const std::string& descriptor, const char* /*cipher*/,
size_t /*len*/, bool /*for_write*/) override; size_t /*len*/, bool /*for_write*/) override;
protected: protected:
Status TEST_Initialize() override;
// PopulateSecretPrefixPart initializes the data into a new prefix block // PopulateSecretPrefixPart initializes the data into a new prefix block
// that will be encrypted. This function will store the data in plain text. // that will be encrypted. This function will store the data in plain text.

71
env/env_test.cc vendored

@ -36,10 +36,13 @@
#endif #endif
#include "env/env_chroot.h" #include "env/env_chroot.h"
#include "env/env_encryption_ctr.h"
#include "logging/log_buffer.h" #include "logging/log_buffer.h"
#include "port/malloc.h" #include "port/malloc.h"
#include "port/port.h" #include "port/port.h"
#include "rocksdb/convenience.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/env_encryption.h"
#include "rocksdb/system_clock.h" #include "rocksdb/system_clock.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "test_util/testharness.h" #include "test_util/testharness.h"
@ -2387,6 +2390,74 @@ TEST_F(EnvTest, EnvWriteVerificationTest) {
ASSERT_OK(s); ASSERT_OK(s);
} }
#ifndef ROCKSDB_LITE
class EncryptionProviderTest : public testing::Test {
public:
};
TEST_F(EncryptionProviderTest, LoadCTRProvider) {
ConfigOptions config_options;
config_options.invoke_prepare_options = false;
std::string CTR = CTREncryptionProvider::kClassName();
std::shared_ptr<EncryptionProvider> provider;
// Test a provider with no cipher
ASSERT_OK(
EncryptionProvider::CreateFromString(config_options, CTR, &provider));
ASSERT_NE(provider, nullptr);
ASSERT_EQ(provider->Name(), CTR);
ASSERT_NOK(provider->PrepareOptions(config_options));
ASSERT_NOK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
auto cipher = provider->GetOptions<std::shared_ptr<BlockCipher>>("Cipher");
ASSERT_NE(cipher, nullptr);
ASSERT_EQ(cipher->get(), nullptr);
provider.reset();
ASSERT_OK(EncryptionProvider::CreateFromString(config_options,
CTR + "://test", &provider));
ASSERT_NE(provider, nullptr);
ASSERT_EQ(provider->Name(), CTR);
ASSERT_OK(provider->PrepareOptions(config_options));
ASSERT_OK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
cipher = provider->GetOptions<std::shared_ptr<BlockCipher>>("Cipher");
ASSERT_NE(cipher, nullptr);
ASSERT_NE(cipher->get(), nullptr);
ASSERT_STREQ(cipher->get()->Name(), "ROT13");
provider.reset();
ASSERT_OK(EncryptionProvider::CreateFromString(config_options, "1://test",
&provider));
ASSERT_NE(provider, nullptr);
ASSERT_EQ(provider->Name(), CTR);
ASSERT_OK(provider->PrepareOptions(config_options));
ASSERT_OK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
cipher = provider->GetOptions<std::shared_ptr<BlockCipher>>("Cipher");
ASSERT_NE(cipher, nullptr);
ASSERT_NE(cipher->get(), nullptr);
ASSERT_STREQ(cipher->get()->Name(), "ROT13");
provider.reset();
ASSERT_OK(EncryptionProvider::CreateFromString(
config_options, "id=" + CTR + "; cipher=ROT13", &provider));
ASSERT_NE(provider, nullptr);
ASSERT_EQ(provider->Name(), CTR);
cipher = provider->GetOptions<std::shared_ptr<BlockCipher>>("Cipher");
ASSERT_NE(cipher, nullptr);
ASSERT_NE(cipher->get(), nullptr);
ASSERT_STREQ(cipher->get()->Name(), "ROT13");
provider.reset();
}
TEST_F(EncryptionProviderTest, LoadROT13Cipher) {
ConfigOptions config_options;
std::shared_ptr<BlockCipher> cipher;
// Test a provider with no cipher
ASSERT_OK(BlockCipher::CreateFromString(config_options, "ROT13", &cipher));
ASSERT_NE(cipher, nullptr);
ASSERT_STREQ(cipher->Name(), "ROT13");
}
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
int main(int argc, char** argv) { int main(int argc, char** argv) {

@ -9,6 +9,7 @@
#include <string> #include <string>
#include "rocksdb/customizable.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/file_system.h" #include "rocksdb/file_system.h"
#include "rocksdb/rocksdb_namespace.h" #include "rocksdb/rocksdb_namespace.h"
@ -58,7 +59,7 @@ class BlockAccessCipherStream {
}; };
// BlockCipher // BlockCipher
class BlockCipher { class BlockCipher : public Customizable {
public: public:
virtual ~BlockCipher(){}; virtual ~BlockCipher(){};
@ -80,12 +81,12 @@ class BlockCipher {
const std::string& value, const std::string& value,
std::shared_ptr<BlockCipher>* result); std::shared_ptr<BlockCipher>* result);
static const char* Type() { return "BlockCipher"; }
// Short-cut method to create a ROT13 BlockCipher. // Short-cut method to create a ROT13 BlockCipher.
// This cipher is only suitable for test purposes and should not be used in // This cipher is only suitable for test purposes and should not be used in
// production!!! // production!!!
static std::shared_ptr<BlockCipher> NewROT13Cipher(size_t block_size); static std::shared_ptr<BlockCipher> NewROT13Cipher(size_t block_size);
virtual const char* Name() const = 0;
// BlockSize returns the size of each block supported by this cipher stream. // BlockSize returns the size of each block supported by this cipher stream.
virtual size_t BlockSize() = 0; virtual size_t BlockSize() = 0;
@ -101,7 +102,7 @@ class BlockCipher {
// The encryption provider is used to create a cipher stream for a specific // The encryption provider is used to create a cipher stream for a specific
// file. The returned cipher stream will be used for actual // file. The returned cipher stream will be used for actual
// encryption/decryption actions. // encryption/decryption actions.
class EncryptionProvider { class EncryptionProvider : public Customizable {
public: public:
virtual ~EncryptionProvider(){}; virtual ~EncryptionProvider(){};
@ -109,14 +110,14 @@ class EncryptionProvider {
// The value describes the type of provider (and potentially optional // The value describes the type of provider (and potentially optional
// configuration parameters) used to create this provider. // configuration parameters) used to create this provider.
// For example, if the value is "CTR", a CTREncryptionProvider will be // For example, if the value is "CTR", a CTREncryptionProvider will be
// created. If the value is preceded by "test://" (e.g test://CTR"), the // created. If the value is ends with "://test" (e.g CTR://test"), the
// TEST_Initialize method will be invoked prior to returning the provider. // provider will be initialized in "TEST" mode prior to being returned.
// //
// @param config_options Options to control how this provider is created // @param config_options Options to control how this provider is created
// and initialized. // and initialized.
// @param value The value might be: // @param value The value might be:
// - CTR Create a CTR provider // - CTR Create a CTR provider
// - test://CTR Create a CTR provider and initialize it for tests. // - CTR://test Create a CTR provider and initialize it for tests.
// @param result The new provider object // @param result The new provider object
// @return OK if the provider was successfully created // @return OK if the provider was successfully created
// @return NotFound if an invalid name was specified in the value // @return NotFound if an invalid name was specified in the value
@ -125,13 +126,12 @@ class EncryptionProvider {
const std::string& value, const std::string& value,
std::shared_ptr<EncryptionProvider>* result); std::shared_ptr<EncryptionProvider>* result);
static const char* Type() { return "EncryptionProvider"; }
// Short-cut method to create a CTR-provider // Short-cut method to create a CTR-provider
static std::shared_ptr<EncryptionProvider> NewCTRProvider( static std::shared_ptr<EncryptionProvider> NewCTRProvider(
const std::shared_ptr<BlockCipher>& cipher); const std::shared_ptr<BlockCipher>& cipher);
// Returns the name of this EncryptionProvider
virtual const char* Name() const = 0;
// GetPrefixLength returns the length of the prefix that is added to every // GetPrefixLength returns the length of the prefix that is added to every
// file and used for storing encryption options. For optimal performance, the // file and used for storing encryption options. For optimal performance, the
// prefix length should be a multiple of the page size. // prefix length should be a multiple of the page size.
@ -165,11 +165,6 @@ class EncryptionProvider {
// or not a file is encrypted by this provider. The maker will also be part // or not a file is encrypted by this provider. The maker will also be part
// of any encryption prefix for this provider. // of any encryption prefix for this provider.
virtual std::string GetMarker() const { return ""; } virtual std::string GetMarker() const { return ""; }
protected:
// Optional method to initialize an EncryptionProvider in the TEST
// environment.
virtual Status TEST_Initialize() { return Status::OK(); }
}; };
class EncryptedSequentialFile : public FSSequentialFile { class EncryptedSequentialFile : public FSSequentialFile {

@ -40,8 +40,9 @@ void Configurable::RegisterOptions(
//************************************************************************* //*************************************************************************
Status Configurable::PrepareOptions(const ConfigOptions& opts) { Status Configurable::PrepareOptions(const ConfigOptions& opts) {
// We ignore the invoke_prepare_options here intentionally,
// as if you are here, you must have called PrepareOptions explicitly.
Status status = Status::OK(); Status status = Status::OK();
if (opts.invoke_prepare_options) {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
for (auto opt_iter : options_) { for (auto opt_iter : options_) {
for (auto map_iter : *(opt_iter.type_map)) { for (auto map_iter : *(opt_iter.type_map)) {
@ -57,18 +58,19 @@ Status Configurable::PrepareOptions(const ConfigOptions& opts) {
return status; return status;
} }
} else if (!opt_info.CanBeNull()) { } else if (!opt_info.CanBeNull()) {
status = Status::NotFound("Missing configurable object", status =
map_iter.first); Status::NotFound("Missing configurable object", map_iter.first);
} }
} }
} }
} }
} }
#else
(void)opts;
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
if (status.ok()) { if (status.ok()) {
is_prepared_ = true; is_prepared_ = true;
} }
}
return status; return status;
} }

@ -18,6 +18,7 @@
#include "options/options_helper.h" #include "options/options_helper.h"
#include "options/options_parser.h" #include "options/options_parser.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/env_encryption.h"
#include "rocksdb/flush_block_policy.h" #include "rocksdb/flush_block_policy.h"
#include "rocksdb/secondary_cache.h" #include "rocksdb/secondary_cache.h"
#include "rocksdb/utilities/customizable_util.h" #include "rocksdb/utilities/customizable_util.h"
@ -27,6 +28,7 @@
#include "table/mock_table.h" #include "table/mock_table.h"
#include "test_util/testharness.h" #include "test_util/testharness.h"
#include "test_util/testutil.h" #include "test_util/testutil.h"
#include "util/string_util.h"
#ifndef GFLAGS #ifndef GFLAGS
bool FLAGS_enable_print = false; bool FLAGS_enable_print = false;
@ -231,15 +233,18 @@ static std::unordered_map<std::string, OptionTypeInfo> simple_option_info = {
{"bool", {"bool",
{offsetof(struct SimpleOptions, b), OptionType::kBoolean, {offsetof(struct SimpleOptions, b), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"unique", OptionTypeInfo::AsCustomUniquePtr<TestCustomizable>( {"unique",
offsetof(struct SimpleOptions, cu), OptionTypeInfo::AsCustomUniquePtr<TestCustomizable>(
OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, offsetof(struct SimpleOptions, cu), OptionVerificationType::kNormal,
{"shared", OptionTypeInfo::AsCustomSharedPtr<TestCustomizable>( OptionTypeFlags::kAllowNull)},
offsetof(struct SimpleOptions, cs), {"shared",
OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, OptionTypeInfo::AsCustomSharedPtr<TestCustomizable>(
{"pointer", OptionTypeInfo::AsCustomRawPtr<TestCustomizable>( offsetof(struct SimpleOptions, cs), OptionVerificationType::kNormal,
offsetof(struct SimpleOptions, cp), OptionTypeFlags::kAllowNull)},
OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, {"pointer",
OptionTypeInfo::AsCustomRawPtr<TestCustomizable>(
offsetof(struct SimpleOptions, cp), OptionVerificationType::kNormal,
OptionTypeFlags::kAllowNull)},
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
}; };
@ -554,11 +559,6 @@ TEST_F(CustomizableTest, PrepareOptionsTest) {
ASSERT_FALSE(simple->cp->IsPrepared()); ASSERT_FALSE(simple->cp->IsPrepared());
ASSERT_OK(base->PrepareOptions(config_options_)); ASSERT_OK(base->PrepareOptions(config_options_));
ASSERT_FALSE(base->IsPrepared());
ASSERT_FALSE(simple->cu->IsPrepared());
ASSERT_FALSE(simple->cs->IsPrepared());
ASSERT_FALSE(simple->cp->IsPrepared());
ASSERT_OK(base->PrepareOptions(prepared));
ASSERT_TRUE(base->IsPrepared()); ASSERT_TRUE(base->IsPrepared());
ASSERT_TRUE(simple->cu->IsPrepared()); ASSERT_TRUE(simple->cu->IsPrepared());
ASSERT_TRUE(simple->cs->IsPrepared()); ASSERT_TRUE(simple->cs->IsPrepared());
@ -917,6 +917,48 @@ static int RegisterTestObjects(ObjectLibrary& library,
return static_cast<int>(library.GetFactoryCount(&num_types)); return static_cast<int>(library.GetFactoryCount(&num_types));
} }
class MockEncryptionProvider : public EncryptionProvider {
public:
explicit MockEncryptionProvider(const std::string& id) : id_(id) {}
const char* Name() const override { return "Mock"; }
size_t GetPrefixLength() const override { return 0; }
Status CreateNewPrefix(const std::string& /*fname*/, char* /*prefix*/,
size_t /*prefixLength*/) const override {
return Status::NotSupported();
}
Status AddCipher(const std::string& /*descriptor*/, const char* /*cipher*/,
size_t /*len*/, bool /*for_write*/) override {
return Status::NotSupported();
}
Status CreateCipherStream(
const std::string& /*fname*/, const EnvOptions& /*options*/,
Slice& /*prefix*/,
std::unique_ptr<BlockAccessCipherStream>* /*result*/) override {
return Status::NotSupported();
}
Status ValidateOptions(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts) const override {
if (EndsWith(id_, "://test")) {
return EncryptionProvider::ValidateOptions(db_opts, cf_opts);
} else {
return Status::InvalidArgument("MockProvider not initialized");
}
}
private:
std::string id_;
};
class MockCipher : public BlockCipher {
public:
const char* Name() const override { return "Mock"; }
size_t BlockSize() override { return 0; }
Status Encrypt(char* /*data*/) override { return Status::NotSupported(); }
Status Decrypt(char* data) override { return Encrypt(data); }
};
class TestFlushBlockPolicyFactory : public FlushBlockPolicyFactory { class TestFlushBlockPolicyFactory : public FlushBlockPolicyFactory {
public: public:
TestFlushBlockPolicyFactory() {} TestFlushBlockPolicyFactory() {}
@ -935,6 +977,19 @@ static int RegisterLocalObjects(ObjectLibrary& library,
const std::string& /*arg*/) { const std::string& /*arg*/) {
size_t num_types; size_t num_types;
// Load any locally defined objects here // Load any locally defined objects here
library.Register<EncryptionProvider>(
"Mock(://test)?",
[](const std::string& uri, std::unique_ptr<EncryptionProvider>* guard,
std::string* /* errmsg */) {
guard->reset(new MockEncryptionProvider(uri));
return guard->get();
});
library.Register<BlockCipher>("Mock", [](const std::string& /*uri*/,
std::unique_ptr<BlockCipher>* guard,
std::string* /* errmsg */) {
guard->reset(new MockCipher());
return guard->get();
});
library.Register<FlushBlockPolicyFactory>( library.Register<FlushBlockPolicyFactory>(
TestFlushBlockPolicyFactory::kClassName(), TestFlushBlockPolicyFactory::kClassName(),
[](const std::string& /*uri*/, [](const std::string& /*uri*/,
@ -957,7 +1012,10 @@ static int RegisterLocalObjects(ObjectLibrary& library,
class LoadCustomizableTest : public testing::Test { class LoadCustomizableTest : public testing::Test {
public: public:
LoadCustomizableTest() { config_options_.ignore_unsupported_options = false; } LoadCustomizableTest() {
config_options_.ignore_unsupported_options = false;
config_options_.invoke_prepare_options = false;
}
bool RegisterTests(const std::string& arg) { bool RegisterTests(const std::string& arg) {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
config_options_.registry->AddLibrary("custom-tests", RegisterTestObjects, config_options_.registry->AddLibrary("custom-tests", RegisterTestObjects,
@ -1047,6 +1105,49 @@ TEST_F(LoadCustomizableTest, LoadComparatorTest) {
} }
} }
#ifndef ROCKSDB_LITE
TEST_F(LoadCustomizableTest, LoadEncryptionProviderTest) {
std::shared_ptr<EncryptionProvider> result;
ASSERT_NOK(
EncryptionProvider::CreateFromString(config_options_, "Mock", &result));
ASSERT_OK(
EncryptionProvider::CreateFromString(config_options_, "CTR", &result));
ASSERT_NE(result, nullptr);
ASSERT_STREQ(result->Name(), "CTR");
ASSERT_NOK(result->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
ASSERT_OK(EncryptionProvider::CreateFromString(config_options_, "CTR://test",
&result));
ASSERT_NE(result, nullptr);
ASSERT_STREQ(result->Name(), "CTR");
ASSERT_OK(result->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
if (RegisterTests("Test")) {
ASSERT_OK(
EncryptionProvider::CreateFromString(config_options_, "Mock", &result));
ASSERT_NE(result, nullptr);
ASSERT_STREQ(result->Name(), "Mock");
ASSERT_OK(EncryptionProvider::CreateFromString(config_options_,
"Mock://test", &result));
ASSERT_NE(result, nullptr);
ASSERT_STREQ(result->Name(), "Mock");
ASSERT_OK(result->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
}
}
TEST_F(LoadCustomizableTest, LoadEncryptionCipherTest) {
std::shared_ptr<BlockCipher> result;
ASSERT_NOK(BlockCipher::CreateFromString(config_options_, "Mock", &result));
ASSERT_OK(BlockCipher::CreateFromString(config_options_, "ROT13", &result));
ASSERT_NE(result, nullptr);
ASSERT_STREQ(result->Name(), "ROT13");
if (RegisterTests("Test")) {
ASSERT_OK(BlockCipher::CreateFromString(config_options_, "Mock", &result));
ASSERT_NE(result, nullptr);
ASSERT_STREQ(result->Name(), "Mock");
}
}
#endif // !ROCKSDB_LITE
TEST_F(LoadCustomizableTest, LoadFlushBlockPolicyFactoryTest) { TEST_F(LoadCustomizableTest, LoadFlushBlockPolicyFactoryTest) {
std::shared_ptr<TableFactory> table; std::shared_ptr<TableFactory> table;
std::shared_ptr<FlushBlockPolicyFactory> result; std::shared_ptr<FlushBlockPolicyFactory> result;

Loading…
Cancel
Save