You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
rocksdb/env/env_basic_test.cc

401 lines
14 KiB

// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
//
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#include <algorithm>
#include <memory>
#include <string>
#include <vector>
#include "env/mock_env.h"
#include "file/file_util.h"
Changes to EncryptedEnv public API (#7279) Summary: Cleaned up the public API to use the EncryptedEnv. This change will allow providers to be developed and added to the system easier in the future. It will also allow better integration in the future with the OPTIONS file. - The internal classes were moved out of the public API into an internal "env_encryption_ctr.h" header. Short-cut constructors were added to provide the original API functionality. - The APIs to the constructors were changed to take shared_ptr, rather than raw pointers or references to allow better memory management and alternative implementations. - CreateFromString methods were added to allow future expansion to other provider and cipher implementations through a standard API. Additionally, there was a code duplication in the NewXXXFile methods. This common code was moved under a templatized function. A first-pass at structuring the code was made to potentially allow multiple EncryptionProviders in a single EncryptedEnv. The idea was that different providers may use different cipher keys or different versions/algorithms. The EncryptedEnv should have some means of picking different providers based on information. The groundwork was started for this (the use of the provider_ member variable was localized) but the work has not been completed. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7279 Reviewed By: jay-zhuang Differential Revision: D23709440 Pulled By: zhichao-cao fbshipit-source-id: 0e845fff0e03a52603eb9672b4ade32d063ff2f2
4 years ago
#include "rocksdb/convenience.h"
#include "rocksdb/env.h"
#include "rocksdb/env_encryption.h"
#include "test_util/testharness.h"
namespace ROCKSDB_NAMESPACE {
namespace {
using CreateEnvFunc = Env*();
// These functions are used to create the various environments under which this
// test can execute. These functions are used to allow the test cases to be
// created without the Env being initialized, thereby eliminating a potential
// static initialization fiasco/race condition when attempting to get a
// custom/configured env prior to main being invoked.
static Env* GetDefaultEnv() { return Env::Default(); }
static Env* GetMockEnv() {
static std::unique_ptr<Env> mock_env(new MockEnv(Env::Default()));
return mock_env.get();
}
#ifndef ROCKSDB_LITE
static Env* NewTestEncryptedEnv(Env* base, const std::string& provider_id) {
ConfigOptions config_opts;
config_opts.invoke_prepare_options = false;
std::shared_ptr<EncryptionProvider> provider;
EXPECT_OK(EncryptionProvider::CreateFromString(config_opts, provider_id,
&provider));
return NewEncryptedEnv(base, provider);
}
static Env* GetCtrEncryptedEnv() {
static std::unique_ptr<Env> ctr_encrypt_env(
NewTestEncryptedEnv(Env::Default(), "CTR://test"));
return ctr_encrypt_env.get();
}
static Env* GetMemoryEnv() {
static std::unique_ptr<Env> mem_env(NewMemEnv(Env::Default()));
return mem_env.get();
}
static Env* GetTestEnv() {
static std::shared_ptr<Env> env_guard;
static Env* custom_env = nullptr;
if (custom_env == nullptr) {
const char* uri = getenv("TEST_ENV_URI");
if (uri != nullptr) {
EXPECT_OK(Env::CreateFromUri(ConfigOptions(), uri, "", &custom_env,
&env_guard));
}
}
EXPECT_NE(custom_env, nullptr);
return custom_env;
}
static Env* GetTestFS() {
static std::shared_ptr<Env> fs_env_guard;
static Env* fs_env = nullptr;
if (fs_env == nullptr) {
const char* uri = getenv("TEST_FS_URI");
if (uri != nullptr) {
EXPECT_OK(
Env::CreateFromUri(ConfigOptions(), uri, "", &fs_env, &fs_env_guard));
}
}
EXPECT_NE(fs_env, nullptr);
return fs_env;
}
#endif // ROCKSDB_LITE
} // namespace
class EnvBasicTestWithParam
: public testing::Test,
public ::testing::WithParamInterface<CreateEnvFunc*> {
public:
Env* env_;
const EnvOptions soptions_;
std::string test_dir_;
EnvBasicTestWithParam() : env_(GetParam()()) {
test_dir_ = test::PerThreadDBPath(env_, "env_basic_test");
}
void SetUp() override { ASSERT_OK(env_->CreateDirIfMissing(test_dir_)); }
void TearDown() override { ASSERT_OK(DestroyDir(env_, test_dir_)); }
};
class EnvMoreTestWithParam : public EnvBasicTestWithParam {};
INSTANTIATE_TEST_CASE_P(EnvDefault, EnvBasicTestWithParam,
::testing::Values(&GetDefaultEnv));
INSTANTIATE_TEST_CASE_P(EnvDefault, EnvMoreTestWithParam,
::testing::Values(&GetDefaultEnv));
INSTANTIATE_TEST_CASE_P(MockEnv, EnvBasicTestWithParam,
::testing::Values(&GetMockEnv));
#ifndef ROCKSDB_LITE
Changes to EncryptedEnv public API (#7279) Summary: Cleaned up the public API to use the EncryptedEnv. This change will allow providers to be developed and added to the system easier in the future. It will also allow better integration in the future with the OPTIONS file. - The internal classes were moved out of the public API into an internal "env_encryption_ctr.h" header. Short-cut constructors were added to provide the original API functionality. - The APIs to the constructors were changed to take shared_ptr, rather than raw pointers or references to allow better memory management and alternative implementations. - CreateFromString methods were added to allow future expansion to other provider and cipher implementations through a standard API. Additionally, there was a code duplication in the NewXXXFile methods. This common code was moved under a templatized function. A first-pass at structuring the code was made to potentially allow multiple EncryptionProviders in a single EncryptedEnv. The idea was that different providers may use different cipher keys or different versions/algorithms. The EncryptedEnv should have some means of picking different providers based on information. The groundwork was started for this (the use of the provider_ member variable was localized) but the work has not been completed. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7279 Reviewed By: jay-zhuang Differential Revision: D23709440 Pulled By: zhichao-cao fbshipit-source-id: 0e845fff0e03a52603eb9672b4ade32d063ff2f2
4 years ago
// next statements run env test against default encryption code.
INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvBasicTestWithParam,
::testing::Values(&GetCtrEncryptedEnv));
INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvMoreTestWithParam,
::testing::Values(&GetCtrEncryptedEnv));
INSTANTIATE_TEST_CASE_P(MemEnv, EnvBasicTestWithParam,
::testing::Values(&GetMemoryEnv));
namespace {
// Returns a vector of 0 or 1 Env*, depending whether an Env is registered for
// TEST_ENV_URI.
//
// The purpose of returning an empty vector (instead of nullptr) is that gtest
// ValuesIn() will skip running tests when given an empty collection.
std::vector<CreateEnvFunc*> GetCustomEnvs() {
std::vector<CreateEnvFunc*> res;
const char* uri = getenv("TEST_ENV_URI");
if (uri != nullptr) {
res.push_back(&GetTestEnv);
}
uri = getenv("TEST_FS_URI");
if (uri != nullptr) {
res.push_back(&GetTestFS);
}
return res;
}
} // anonymous namespace
INSTANTIATE_TEST_CASE_P(CustomEnv, EnvBasicTestWithParam,
::testing::ValuesIn(GetCustomEnvs()));
INSTANTIATE_TEST_CASE_P(CustomEnv, EnvMoreTestWithParam,
::testing::ValuesIn(GetCustomEnvs()));
#endif // ROCKSDB_LITE
TEST_P(EnvBasicTestWithParam, Basics) {
uint64_t file_size;
std::unique_ptr<WritableFile> writable_file;
std::vector<std::string> children;
// Check that the directory is empty.
ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/non_existent"));
ASSERT_TRUE(!env_->GetFileSize(test_dir_ + "/non_existent", &file_size).ok());
ASSERT_OK(env_->GetChildren(test_dir_, &children));
ASSERT_EQ(0U, children.size());
// Create a file.
ASSERT_OK(env_->NewWritableFile(test_dir_ + "/f", &writable_file, soptions_));
ASSERT_OK(writable_file->Close());
writable_file.reset();
// Check that the file exists.
ASSERT_OK(env_->FileExists(test_dir_ + "/f"));
ASSERT_OK(env_->GetFileSize(test_dir_ + "/f", &file_size));
ASSERT_EQ(0U, file_size);
ASSERT_OK(env_->GetChildren(test_dir_, &children));
ASSERT_EQ(1U, children.size());
ASSERT_EQ("f", children[0]);
ASSERT_OK(env_->DeleteFile(test_dir_ + "/f"));
// Write to the file.
ASSERT_OK(
env_->NewWritableFile(test_dir_ + "/f1", &writable_file, soptions_));
ASSERT_OK(writable_file->Append("abc"));
ASSERT_OK(writable_file->Close());
writable_file.reset();
ASSERT_OK(
env_->NewWritableFile(test_dir_ + "/f2", &writable_file, soptions_));
ASSERT_OK(writable_file->Close());
writable_file.reset();
// Check for expected size.
ASSERT_OK(env_->GetFileSize(test_dir_ + "/f1", &file_size));
ASSERT_EQ(3U, file_size);
// Check that renaming works.
ASSERT_TRUE(
!env_->RenameFile(test_dir_ + "/non_existent", test_dir_ + "/g").ok());
ASSERT_OK(env_->RenameFile(test_dir_ + "/f1", test_dir_ + "/g"));
ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/f1"));
ASSERT_OK(env_->FileExists(test_dir_ + "/g"));
ASSERT_OK(env_->GetFileSize(test_dir_ + "/g", &file_size));
ASSERT_EQ(3U, file_size);
// Check that renaming overwriting works
ASSERT_OK(env_->RenameFile(test_dir_ + "/f2", test_dir_ + "/g"));
ASSERT_OK(env_->GetFileSize(test_dir_ + "/g", &file_size));
ASSERT_EQ(0U, file_size);
// Check that opening non-existent file fails.
std::unique_ptr<SequentialFile> seq_file;
std::unique_ptr<RandomAccessFile> rand_file;
ASSERT_TRUE(!env_->NewSequentialFile(test_dir_ + "/non_existent", &seq_file,
soptions_)
.ok());
ASSERT_TRUE(!seq_file);
ASSERT_NOK(env_->NewRandomAccessFile(test_dir_ + "/non_existent", &rand_file,
soptions_));
ASSERT_TRUE(!rand_file);
// Check that deleting works.
ASSERT_NOK(env_->DeleteFile(test_dir_ + "/non_existent"));
ASSERT_OK(env_->DeleteFile(test_dir_ + "/g"));
ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/g"));
ASSERT_OK(env_->GetChildren(test_dir_, &children));
ASSERT_EQ(0U, children.size());
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566) Summary: This PR does a few things: 1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation. 2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed: - The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated - The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory). 3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10). I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged. Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently. Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :) Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566 Reviewed By: zhichao-cao Differential Revision: D24408980 Pulled By: jay-zhuang fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
4 years ago
Status s = env_->GetChildren(test_dir_ + "/non_existent", &children);
ASSERT_TRUE(s.IsNotFound());
}
TEST_P(EnvBasicTestWithParam, ReadWrite) {
std::unique_ptr<WritableFile> writable_file;
std::unique_ptr<SequentialFile> seq_file;
std::unique_ptr<RandomAccessFile> rand_file;
Slice result;
char scratch[100];
ASSERT_OK(env_->NewWritableFile(test_dir_ + "/f", &writable_file, soptions_));
ASSERT_OK(writable_file->Append("hello "));
ASSERT_OK(writable_file->Append("world"));
ASSERT_OK(writable_file->Close());
writable_file.reset();
// Read sequentially.
ASSERT_OK(env_->NewSequentialFile(test_dir_ + "/f", &seq_file, soptions_));
ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
ASSERT_EQ(0, result.compare("hello"));
ASSERT_OK(seq_file->Skip(1));
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world".
ASSERT_EQ(0, result.compare("world"));
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF.
ASSERT_EQ(0U, result.size());
ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file.
ASSERT_OK(seq_file->Read(1000, &result, scratch));
ASSERT_EQ(0U, result.size());
// Random reads.
ASSERT_OK(env_->NewRandomAccessFile(test_dir_ + "/f", &rand_file, soptions_));
ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
ASSERT_EQ(0, result.compare("world"));
ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
ASSERT_EQ(0, result.compare("hello"));
ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d".
ASSERT_EQ(0, result.compare("d"));
// Too high offset.
ASSERT_TRUE(rand_file->Read(1000, 5, &result, scratch).ok());
}
TEST_P(EnvBasicTestWithParam, Misc) {
std::unique_ptr<WritableFile> writable_file;
ASSERT_OK(env_->NewWritableFile(test_dir_ + "/b", &writable_file, soptions_));
// These are no-ops, but we test they return success.
ASSERT_OK(writable_file->Sync());
ASSERT_OK(writable_file->Flush());
ASSERT_OK(writable_file->Close());
writable_file.reset();
}
TEST_P(EnvBasicTestWithParam, LargeWrite) {
const size_t kWriteSize = 300 * 1024;
char* scratch = new char[kWriteSize * 2];
std::string write_data;
for (size_t i = 0; i < kWriteSize; ++i) {
write_data.append(1, static_cast<char>(i));
}
std::unique_ptr<WritableFile> writable_file;
ASSERT_OK(env_->NewWritableFile(test_dir_ + "/f", &writable_file, soptions_));
ASSERT_OK(writable_file->Append("foo"));
ASSERT_OK(writable_file->Append(write_data));
ASSERT_OK(writable_file->Close());
writable_file.reset();
std::unique_ptr<SequentialFile> seq_file;
Slice result;
ASSERT_OK(env_->NewSequentialFile(test_dir_ + "/f", &seq_file, soptions_));
ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
ASSERT_EQ(0, result.compare("foo"));
size_t read = 0;
std::string read_data;
while (read < kWriteSize) {
ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch));
read_data.append(result.data(), result.size());
read += result.size();
}
ASSERT_TRUE(write_data == read_data);
delete [] scratch;
}
TEST_P(EnvMoreTestWithParam, GetModTime) {
ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/dir1"));
uint64_t mtime1 = 0x0;
ASSERT_OK(env_->GetFileModificationTime(test_dir_ + "/dir1", &mtime1));
}
TEST_P(EnvMoreTestWithParam, MakeDir) {
ASSERT_OK(env_->CreateDir(test_dir_ + "/j"));
ASSERT_OK(env_->FileExists(test_dir_ + "/j"));
std::vector<std::string> children;
ASSERT_OK(env_->GetChildren(test_dir_, &children));
ASSERT_EQ(1U, children.size());
// fail because file already exists
ASSERT_TRUE(!env_->CreateDir(test_dir_ + "/j").ok());
ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/j"));
ASSERT_OK(env_->DeleteDir(test_dir_ + "/j"));
ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/j"));
}
TEST_P(EnvMoreTestWithParam, GetChildren) {
// empty folder returns empty vector
std::vector<std::string> children;
std::vector<Env::FileAttributes> childAttr;
ASSERT_OK(env_->CreateDirIfMissing(test_dir_));
ASSERT_OK(env_->GetChildren(test_dir_, &children));
ASSERT_OK(env_->FileExists(test_dir_));
ASSERT_OK(env_->GetChildrenFileAttributes(test_dir_, &childAttr));
ASSERT_EQ(0U, children.size());
ASSERT_EQ(0U, childAttr.size());
// folder with contents returns relative path to test dir
ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/niu"));
ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/you"));
ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/guo"));
ASSERT_OK(env_->GetChildren(test_dir_, &children));
ASSERT_OK(env_->GetChildrenFileAttributes(test_dir_, &childAttr));
ASSERT_EQ(3U, children.size());
ASSERT_EQ(3U, childAttr.size());
for (auto each : children) {
env_->DeleteDir(test_dir_ + "/" + each).PermitUncheckedError();
} // necessary for default POSIX env
// non-exist directory returns IOError
ASSERT_OK(env_->DeleteDir(test_dir_));
ASSERT_NOK(env_->FileExists(test_dir_));
ASSERT_NOK(env_->GetChildren(test_dir_, &children));
ASSERT_NOK(env_->GetChildrenFileAttributes(test_dir_, &childAttr));
// if dir is a file, returns IOError
ASSERT_OK(env_->CreateDir(test_dir_));
std::unique_ptr<WritableFile> writable_file;
ASSERT_OK(
env_->NewWritableFile(test_dir_ + "/file", &writable_file, soptions_));
ASSERT_OK(writable_file->Close());
writable_file.reset();
ASSERT_NOK(env_->GetChildren(test_dir_ + "/file", &children));
ASSERT_EQ(0U, children.size());
}
TEST_P(EnvMoreTestWithParam, GetChildrenIgnoresDotAndDotDot) {
auto* env = Env::Default();
ASSERT_OK(env->CreateDirIfMissing(test_dir_));
// Create a single file
std::string path = test_dir_;
const EnvOptions soptions;
#ifdef OS_WIN
path.append("\\test_file");
#else
path.append("/test_file");
#endif
std::string data("test data");
std::unique_ptr<WritableFile> file;
ASSERT_OK(env->NewWritableFile(path, &file, soptions));
ASSERT_OK(file->Append("test data"));
// get the children
std::vector<std::string> result;
ASSERT_OK(env->GetChildren(test_dir_, &result));
// expect only one file named `test_data`, i.e. no `.` or `..` names
ASSERT_EQ(result.size(), 1);
ASSERT_EQ(result.at(0), "test_file");
}
} // namespace ROCKSDB_NAMESPACE
int main(int argc, char** argv) {
rocksdb: switch to gtest Summary: Our existing test notation is very similar to what is used in gtest. It makes it easy to adopt what is different. In this diff I modify existing [[ https://code.google.com/p/googletest/wiki/Primer#Test_Fixtures:_Using_the_Same_Data_Configuration_for_Multiple_Te | test fixture ]] classes to inherit from `testing::Test`. Also for unit tests that use fixture class, `TEST` is replaced with `TEST_F` as required in gtest. There are several custom `main` functions in our existing tests. To make this transition easier, I modify all `main` functions to fallow gtest notation. But eventually we can remove them and use implementation of `main` that gtest provides. ```lang=bash % cat ~/transform #!/bin/sh files=$(git ls-files '*test\.cc') for file in $files do if grep -q "rocksdb::test::RunAllTests()" $file then if grep -Eq '^class \w+Test {' $file then perl -pi -e 's/^(class \w+Test) {/${1}: public testing::Test {/g' $file perl -pi -e 's/^(TEST)/${1}_F/g' $file fi perl -pi -e 's/(int main.*\{)/${1}::testing::InitGoogleTest(&argc, argv);/g' $file perl -pi -e 's/rocksdb::test::RunAllTests/RUN_ALL_TESTS/g' $file fi done % sh ~/transform % make format ``` Second iteration of this diff contains only scripted changes. Third iteration contains manual changes to fix last errors and make it compilable. Test Plan: Build and notice no errors. ```lang=bash % USE_CLANG=1 make check -j55 ``` Tests are still testing. Reviewers: meyering, sdong, rven, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D35157
10 years ago
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}