Introduce chroot Env

Summary:
For testing backups, we needed an Env that is fully isolated from other
Envs on the same machine. Our in-memory Envs (MockEnv and InMemoryEnv) were
insufficient because they don't implement most directory operations.

This diff introduces a new Env, "ChrootEnv", that translates paths such that the
chroot directory appears to be the root directory. This way, multiple Envs can
be isolated in the filesystem by using different chroot directories. Since we
use the filesystem, all directory operations are trivially supported.

Test Plan:
I parameterized the existing EnvPosixTest so it runs tests on ChrootEnv
except the ioctl-related cases.

Reviewers: sdong, lightmark, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D57543

CMakeLists.txt

@@ -200,6 +200,7 @@ set(SOURCES
         util/delete_scheduler.cc
         util/dynamic_bloom.cc
         util/env.cc
+        util/env_chroot.cc
         util/env_hdfs.cc
         util/event_logger.cc
         util/file_util.cc

src.mk

@@ -97,6 +97,7 @@ LIB_SOURCES =                                                   \
   util/delete_scheduler.cc                                      \
   util/dynamic_bloom.cc                                         \
   util/env.cc                                                   \
+  util/env_chroot.cc                                            \
   util/env_hdfs.cc                                              \
   util/env_posix.cc                                             \
   util/io_posix.cc                                              \

util/env_chroot.cc

@@ -0,0 +1,290 @@
+//  Copyright (c) 2016-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#if !defined(ROCKSDB_LITE) && !defined(OS_WIN)
+
+#include "util/env_chroot.h"
+
+#include <unistd.h>
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "rocksdb/status.h"
+
+namespace rocksdb {
+
+class ChrootEnv : public EnvWrapper {
+ public:
+  ChrootEnv(Env* base_env, const std::string& chroot_dir)
+      : EnvWrapper(base_env), chroot_dir_(chroot_dir) {}
+
+  virtual Status NewSequentialFile(const std::string& fname,
+                                   std::unique_ptr<SequentialFile>* result,
+                                   const EnvOptions& options) override {
+    auto status_and_enc_path = EncodePathWithNewBasename(fname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::NewSequentialFile(status_and_enc_path.second, result,
+                                         options);
+  }
+
+  virtual Status NewRandomAccessFile(const std::string& fname,
+                                     unique_ptr<RandomAccessFile>* result,
+                                     const EnvOptions& options) override {
+    auto status_and_enc_path = EncodePathWithNewBasename(fname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::NewRandomAccessFile(status_and_enc_path.second, result,
+                                           options);
+  }
+
+  virtual Status NewWritableFile(const std::string& fname,
+                                 unique_ptr<WritableFile>* result,
+                                 const EnvOptions& options) override {
+    auto status_and_enc_path = EncodePathWithNewBasename(fname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::NewWritableFile(status_and_enc_path.second, result,
+                                       options);
+  }
+
+  virtual Status ReuseWritableFile(const std::string& fname,
+                                   const std::string& old_fname,
+                                   unique_ptr<WritableFile>* result,
+                                   const EnvOptions& options) override {
+    auto status_and_enc_path = EncodePathWithNewBasename(fname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    auto status_and_old_enc_path = EncodePath(old_fname);
+    if (!status_and_old_enc_path.first.ok()) {
+      return status_and_old_enc_path.first;
+    }
+    return EnvWrapper::ReuseWritableFile(status_and_old_enc_path.second,
+                                         status_and_old_enc_path.second, result,
+                                         options);
+  }
+
+  virtual Status NewDirectory(const std::string& dir,
+                              unique_ptr<Directory>* result) override {
+    auto status_and_enc_path = EncodePathWithNewBasename(dir);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::NewDirectory(status_and_enc_path.second, result);
+  }
+
+  virtual Status FileExists(const std::string& fname) override {
+    auto status_and_enc_path = EncodePathWithNewBasename(fname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::FileExists(status_and_enc_path.second);
+  }
+
+  virtual Status GetChildren(const std::string& dir,
+                             std::vector<std::string>* result) override {
+    auto status_and_enc_path = EncodePath(dir);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::GetChildren(status_and_enc_path.second, result);
+  }
+
+  virtual Status GetChildrenFileAttributes(
+      const std::string& dir, std::vector<FileAttributes>* result) override {
+    auto status_and_enc_path = EncodePath(dir);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::GetChildrenFileAttributes(status_and_enc_path.second,
+                                                 result);
+  }
+
+  virtual Status DeleteFile(const std::string& fname) override {
+    auto status_and_enc_path = EncodePath(fname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::DeleteFile(status_and_enc_path.second);
+  }
+
+  virtual Status CreateDir(const std::string& dirname) override {
+    auto status_and_enc_path = EncodePathWithNewBasename(dirname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::CreateDir(status_and_enc_path.second);
+  }
+
+  virtual Status CreateDirIfMissing(const std::string& dirname) override {
+    auto status_and_enc_path = EncodePathWithNewBasename(dirname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::CreateDirIfMissing(status_and_enc_path.second);
+  }
+
+  virtual Status DeleteDir(const std::string& dirname) override {
+    auto status_and_enc_path = EncodePath(dirname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::DeleteDir(status_and_enc_path.second);
+  }
+
+  virtual Status GetFileSize(const std::string& fname,
+                             uint64_t* file_size) override {
+    auto status_and_enc_path = EncodePath(fname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::GetFileSize(status_and_enc_path.second, file_size);
+  }
+
+  virtual Status GetFileModificationTime(const std::string& fname,
+                                         uint64_t* file_mtime) override {
+    auto status_and_enc_path = EncodePath(fname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::GetFileModificationTime(status_and_enc_path.second,
+                                               file_mtime);
+  }
+
+  virtual Status RenameFile(const std::string& src,
+                            const std::string& dest) override {
+    auto status_and_src_enc_path = EncodePath(src);
+    if (!status_and_src_enc_path.first.ok()) {
+      return status_and_src_enc_path.first;
+    }
+    auto status_and_dest_enc_path = EncodePathWithNewBasename(dest);
+    if (!status_and_dest_enc_path.first.ok()) {
+      return status_and_dest_enc_path.first;
+    }
+    return EnvWrapper::RenameFile(status_and_src_enc_path.second,
+                                  status_and_dest_enc_path.second);
+  }
+
+  virtual Status LinkFile(const std::string& src,
+                          const std::string& dest) override {
+    auto status_and_src_enc_path = EncodePath(src);
+    if (!status_and_src_enc_path.first.ok()) {
+      return status_and_src_enc_path.first;
+    }
+    auto status_and_dest_enc_path = EncodePathWithNewBasename(dest);
+    if (!status_and_dest_enc_path.first.ok()) {
+      return status_and_dest_enc_path.first;
+    }
+    return EnvWrapper::LinkFile(status_and_src_enc_path.second,
+                                status_and_dest_enc_path.second);
+  }
+
+  virtual Status LockFile(const std::string& fname, FileLock** lock) override {
+    auto status_and_enc_path = EncodePathWithNewBasename(fname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    // FileLock subclasses may store path (e.g., PosixFileLock stores it). We
+    // can skip stripping the chroot directory from this path because callers
+    // shouldn't use it.
+    return EnvWrapper::LockFile(status_and_enc_path.second, lock);
+  }
+
+  virtual Status GetTestDirectory(std::string* path) override {
+    // Adapted from PosixEnv's implementation since it doesn't provide a way to
+    // create directory in the chroot.
+    char buf[256];
+    snprintf(buf, sizeof(buf), "/rocksdbtest-%d", static_cast<int>(geteuid()));
+    *path = buf;
+
+    // Directory may already exist, so ignore return
+    CreateDir(*path);
+    return Status::OK();
+  }
+
+  virtual Status NewLogger(const std::string& fname,
+                           shared_ptr<Logger>* result) override {
+    auto status_and_enc_path = EncodePathWithNewBasename(fname);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::NewLogger(status_and_enc_path.second, result);
+  }
+
+  virtual Status GetAbsolutePath(const std::string& db_path,
+                                 std::string* output_path) override {
+    auto status_and_enc_path = EncodePath(db_path);
+    if (!status_and_enc_path.first.ok()) {
+      return status_and_enc_path.first;
+    }
+    return EnvWrapper::GetAbsolutePath(status_and_enc_path.second, output_path);
+  }
+
+ private:
+  // Returns status and expanded absolute path including the chroot directory.
+  // Checks whether the provided path breaks out of the chroot. If it returns
+  // non-OK status, the returned path should not be used.
+  std::pair<Status, std::string> EncodePath(const std::string& path) {
+    if (path.empty() || path[0] != '/') {
+      return {Status::InvalidArgument(path, "Not an absolute path"), ""};
+    }
+    std::pair<Status, std::string> res;
+    res.second = chroot_dir_ + path;
+    char* normalized_path = realpath(res.second.c_str(), nullptr);
+    if (normalized_path == nullptr) {
+      res.first = Status::NotFound(res.second, strerror(errno));
+    } else if (strlen(normalized_path) < chroot_dir_.size() ||
+               strncmp(normalized_path, chroot_dir_.c_str(),
+                       chroot_dir_.size()) != 0) {
+      res.first = Status::IOError(res.second,
+                                  "Attempted to access path outside chroot");
+    } else {
+      res.first = Status::OK();
+    }
+    free(normalized_path);
+    return res;
+  }
+
+  // Similar to EncodePath() except assumes the basename in the path hasn't been
+  // created yet.
+  std::pair<Status, std::string> EncodePathWithNewBasename(
+      const std::string& path) {
+    if (path.empty() || path[0] != '/') {
+      return {Status::InvalidArgument(path, "Not an absolute path"), ""};
+    }
+    // Basename may be followed by trailing slashes
+    size_t final_idx = path.find_last_not_of('/');
+    if (final_idx == std::string::npos) {
+      // It's only slashes so no basename to extract
+      return EncodePath(path);
+    }
+
+    // Pull off the basename temporarily since realname(3) (used by
+    // EncodePath()) requires a path that exists
+    size_t base_sep = path.rfind('/', final_idx);
+    auto status_and_enc_path = EncodePath(path.substr(0, base_sep + 1));
+    status_and_enc_path.second.append(path.substr(base_sep + 1));
+    return status_and_enc_path;
+  }
+
+  const std::string chroot_dir_;
+};
+
+Env* NewChrootEnv(Env* base_env, const std::string& chroot_dir) {
+  if (!base_env->FileExists(chroot_dir).ok()) {
+    return nullptr;
+  }
+  return new ChrootEnv(base_env, chroot_dir);
+}
+
+}  // namespace rocksdb
+
+#endif  // !defined(ROCKSDB_LITE) && !defined(OS_WIN)

util/env_chroot.h

@@ -0,0 +1,22 @@
+//  Copyright (c) 2016-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#pragma once
+
+#if !defined(ROCKSDB_LITE) && !defined(OS_WIN)
+
+#include <string>
+
+#include "rocksdb/env.h"
+
+namespace rocksdb {
+
+// Returns an Env that translates paths such that the root directory appears to
+// be chroot_dir. chroot_dir should refer to an existing directory.
+Env* NewChrootEnv(Env* base_env, const std::string& chroot_dir);
+
+}  // namespace rocksdb
+
+#endif  // !defined(ROCKSDB_LITE) && !defined(OS_WIN)

util/env_test.cc

@@ -29,9 +29,10 @@
 #include <errno.h>
 #endif
 
-#include "rocksdb/env.h"
 #include "port/port.h"
+#include "rocksdb/env.h"
 #include "util/coding.h"
+#include "util/env_chroot.h"
 #include "util/log_buffer.h"
 #include "util/mutexlock.h"
 #include "util/string_util.h"
@@ -52,19 +53,25 @@ class EnvPosixTest : public testing::Test {
   EnvPosixTest() : env_(Env::Default()) { }
 };
 
+class EnvPosixTestWithParam : public EnvPosixTest,
+                              public ::testing::WithParamInterface<Env*> {
+ public:
+  EnvPosixTestWithParam() { env_ = GetParam(); }
+};
+
 static void SetBool(void* ptr) {
   reinterpret_cast<std::atomic<bool>*>(ptr)
       ->store(true, std::memory_order_relaxed);
 }
 
-TEST_F(EnvPosixTest, RunImmediately) {
+TEST_P(EnvPosixTestWithParam, RunImmediately) {
   std::atomic<bool> called(false);
   env_->Schedule(&SetBool, &called);
   Env::Default()->SleepForMicroseconds(kDelayMicros);
   ASSERT_TRUE(called.load(std::memory_order_relaxed));
 }
 
-TEST_F(EnvPosixTest, UnSchedule) {
+TEST_P(EnvPosixTestWithParam, UnSchedule) {
   std::atomic<bool> called(false);
   env_->SetBackgroundThreads(1, Env::LOW);
 
@@ -99,7 +106,7 @@ TEST_F(EnvPosixTest, UnSchedule) {
   ASSERT_TRUE(!sleeping_task.IsSleeping() && !sleeping_task1.IsSleeping());
 }
 
-TEST_F(EnvPosixTest, RunMany) {
+TEST_P(EnvPosixTestWithParam, RunMany) {
   std::atomic<int> last_id(0);
 
   struct CB {
@@ -145,7 +152,7 @@ static void ThreadBody(void* arg) {
   s->mu.Unlock();
 }
 
-TEST_F(EnvPosixTest, StartThread) {
+TEST_P(EnvPosixTestWithParam, StartThread) {
   State state;
   state.val = 0;
   state.num_running = 3;
@@ -164,7 +171,7 @@ TEST_F(EnvPosixTest, StartThread) {
   ASSERT_EQ(state.val, 3);
 }
 
-TEST_F(EnvPosixTest, TwoPools) {
+TEST_P(EnvPosixTestWithParam, TwoPools) {
   class CB {
    public:
     CB(const std::string& pool_name, int pool_size)
@@ -281,7 +288,7 @@ TEST_F(EnvPosixTest, TwoPools) {
   env_->SetBackgroundThreads(kHighPoolSize, Env::Priority::HIGH);
 }
 
-TEST_F(EnvPosixTest, DecreaseNumBgThreads) {
+TEST_P(EnvPosixTestWithParam, DecreaseNumBgThreads) {
   std::vector<test::SleepingBackgroundTask> tasks(10);
 
   // Set number of thread to 1 first.
@@ -743,9 +750,9 @@ TEST_F(EnvPosixTest, RandomAccessUniqueIDDeletes) {
 }
 
 // Only works in linux platforms
-TEST_F(EnvPosixTest, InvalidateCache) {
+TEST_P(EnvPosixTestWithParam, InvalidateCache) {
   const EnvOptions soptions;
-  std::string fname = test::TmpDir() + "/" + "testfile";
+  std::string fname = test::TmpDir(env_) + "/" + "testfile";
 
   // Create file.
   {
@@ -828,7 +835,7 @@ class TestLogger : public Logger {
   int char_0_count;
 };
 
-TEST_F(EnvPosixTest, LogBufferTest) {
+TEST_P(EnvPosixTestWithParam, LogBufferTest) {
   TestLogger test_logger;
   test_logger.SetInfoLogLevel(InfoLogLevel::INFO_LEVEL);
   test_logger.log_count = 0;
@@ -886,7 +893,7 @@ class TestLogger2 : public Logger {
   size_t max_log_size_;
 };
 
-TEST_F(EnvPosixTest, LogBufferMaxSizeTest) {
+TEST_P(EnvPosixTestWithParam, LogBufferMaxSizeTest) {
   char bytes9000[9000];
   std::fill_n(bytes9000, sizeof(bytes9000), '1');
   bytes9000[sizeof(bytes9000) - 1] = '\0';
@@ -901,8 +908,8 @@ TEST_F(EnvPosixTest, LogBufferMaxSizeTest) {
   }
 }
 
-TEST_F(EnvPosixTest, Preallocation) {
+TEST_P(EnvPosixTestWithParam, Preallocation) {
-  const std::string src = test::TmpDir() + "/" + "testfile";
+  const std::string src = test::TmpDir(env_) + "/" + "testfile";
   unique_ptr<WritableFile> srcfile;
   const EnvOptions soptions;
   ASSERT_OK(env_->NewWritableFile(src, &srcfile, soptions));
@@ -937,14 +944,14 @@ TEST_F(EnvPosixTest, Preallocation) {
 
 // Test that the two ways to get children file attributes (in bulk or
 // individually) behave consistently.
-TEST_F(EnvPosixTest, ConsistentChildrenAttributes) {
+TEST_P(EnvPosixTestWithParam, ConsistentChildrenAttributes) {
   const EnvOptions soptions;
   const int kNumChildren = 10;
 
   std::string data;
   for (int i = 0; i < kNumChildren; ++i) {
     std::ostringstream oss;
-    oss << test::TmpDir() << "/testfile_" << i;
+    oss << test::TmpDir(env_) << "/testfile_" << i;
     const std::string path = oss.str();
     unique_ptr<WritableFile> file;
     ASSERT_OK(env_->NewWritableFile(path, &file, soptions));
@@ -953,12 +960,12 @@ TEST_F(EnvPosixTest, ConsistentChildrenAttributes) {
   }
 
   std::vector<Env::FileAttributes> file_attrs;
-  ASSERT_OK(env_->GetChildrenFileAttributes(test::TmpDir(), &file_attrs));
+  ASSERT_OK(env_->GetChildrenFileAttributes(test::TmpDir(env_), &file_attrs));
   for (int i = 0; i < kNumChildren; ++i) {
     std::ostringstream oss;
     oss << "testfile_" << i;
     const std::string name = oss.str();
-    const std::string path = test::TmpDir() + "/" + name;
+    const std::string path = test::TmpDir(env_) + "/" + name;
 
     auto file_attrs_iter = std::find_if(
         file_attrs.begin(), file_attrs.end(),
@@ -972,7 +979,7 @@ TEST_F(EnvPosixTest, ConsistentChildrenAttributes) {
 }
 
 // Test that all WritableFileWrapper forwards all calls to WritableFile.
-TEST_F(EnvPosixTest, WritableFileWrapper) {
+TEST_P(EnvPosixTestWithParam, WritableFileWrapper) {
   class Base : public WritableFile {
    public:
     mutable int *step_;
@@ -1053,6 +1060,14 @@ TEST_F(EnvPosixTest, WritableFileWrapper) {
   EXPECT_EQ(14, step);
 }
 
+INSTANTIATE_TEST_CASE_P(DefaultEnv, EnvPosixTestWithParam,
+                        ::testing::Values(Env::Default()));
+#if !defined(ROCKSDB_LITE) && !defined(OS_WIN)
+INSTANTIATE_TEST_CASE_P(ChrootEnv, EnvPosixTestWithParam,
+                        ::testing::Values(NewChrootEnv(Env::Default(),
+                                                       "/tmp")));
+#endif  // !defined(ROCKSDB_LITE) && !defined(OS_WIN)
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {