Add CreateFrom methods to Env/FileSystem (#8174)

Summary:
- Added CreateFromString method to Env and FilesSystem to replace LoadEnv/Load.  This method/signature is a precursor to making these classes extend Customizable.

- Added CreateFromSystem to Env.  This method standardizes creating an Env from the environment variables.  Previously, some places would check TEST_ENV_URI and others would also check TEST_FS_URI.  Now the code is more command/standardized.

- Added CreateFromFlags to Env.  These method allows Env to be create from string options (such as GFLAGS options) in a more standard way.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8174

Reviewed By: zhichao-cao

Differential Revision: D28999603

Pulled By: mrambacher

fbshipit-source-id: 88e6911e7e91f908458a7fe10a20e93ecbc275fb

db/column_family_test.cc

@@ -56,16 +56,8 @@ class ColumnFamilyTestBase : public testing::Test {
  public:
   explicit ColumnFamilyTestBase(uint32_t format) : rnd_(139), format_(format) {
     Env* base_env = Env::Default();
-#ifndef ROCKSDB_LITE
+    EXPECT_OK(
-    const char* test_env_uri = getenv("TEST_ENV_URI");
+        test::CreateEnvFromSystem(ConfigOptions(), &base_env, &env_guard_));
-    if (test_env_uri) {
-      Env* test_env = nullptr;
-      Status s = Env::LoadEnv(test_env_uri, &test_env, &env_guard_);
-      base_env = test_env;
-      EXPECT_OK(s);
-      EXPECT_NE(Env::Default(), base_env);
-    }
-#endif  // !ROCKSDB_LITE
     EXPECT_NE(nullptr, base_env);
     env_ = new EnvCounter(base_env);
     env_->skip_fsync_ = true;

db/corruption_test.cc

@@ -55,14 +55,9 @@ class CorruptionTest : public testing::Test {
     // bug in recovery code. Keep it 4 for now to make the test passes.
     tiny_cache_ = NewLRUCache(100, 4);
     Env* base_env = Env::Default();
-#ifndef ROCKSDB_LITE
+    EXPECT_OK(
-    const char* test_env_uri = getenv("TEST_ENV_URI");
+        test::CreateEnvFromSystem(ConfigOptions(), &base_env, &env_guard_));
-    if (test_env_uri) {
+    EXPECT_NE(base_env, nullptr);
-      Status s = Env::LoadEnv(test_env_uri, &base_env, &env_guard_);
-      EXPECT_OK(s);
-      EXPECT_NE(Env::Default(), base_env);
-    }
-#endif  //! ROCKSDB_LITE
     env_ = new test::ErrorEnv(base_env);
     options_.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
     options_.env = env_;

db/db_test_util.cc

@@ -59,16 +59,8 @@ SpecialEnv::SpecialEnv(Env* base, bool time_elapse_only_sleep)
 DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
     : mem_env_(nullptr), encrypted_env_(nullptr), option_config_(kDefault) {
   Env* base_env = Env::Default();
-#ifndef ROCKSDB_LITE
+  ConfigOptions config_options;
-  const char* test_env_uri = getenv("TEST_ENV_URI");
+  EXPECT_OK(test::CreateEnvFromSystem(config_options, &base_env, &env_guard_));
-  if (test_env_uri) {
-    Env* test_env = nullptr;
-    Status s = Env::LoadEnv(test_env_uri, &test_env, &env_guard_);
-    base_env = test_env;
-    EXPECT_OK(s);
-    EXPECT_NE(Env::Default(), base_env);
-  }
-#endif  // !ROCKSDB_LITE
   EXPECT_NE(nullptr, base_env);
   if (getenv("MEM_ENV")) {
     mem_env_ = new MockEnv(base_env);
@@ -77,7 +69,7 @@ DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
   if (getenv("ENCRYPTED_ENV")) {
     std::shared_ptr<EncryptionProvider> provider;
     Status s = EncryptionProvider::CreateFromString(
-        ConfigOptions(), std::string("test://") + getenv("ENCRYPTED_ENV"),
+        config_options, std::string("test://") + getenv("ENCRYPTED_ENV"),
         &provider);
     encrypted_env_ = NewEncryptedEnv(mem_env_ ? mem_env_ : base_env, provider);
   }

db/version_set_test.cc

@@ -11,6 +11,7 @@
 
 #include "db/db_impl/db_impl.h"
 #include "db/log_writer.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/file_system.h"
 #include "table/block_based/block_based_table_factory.h"
 #include "table/mock_table.h"
@@ -704,15 +705,10 @@ class VersionSetTestBase {
         write_buffer_manager_(db_options_.db_write_buffer_size),
         shutting_down_(false),
         mock_table_factory_(std::make_shared<mock::MockTableFactory>()) {
-    const char* test_env_uri = getenv("TEST_ENV_URI");
+    EXPECT_OK(test::CreateEnvFromSystem(ConfigOptions(), &env_, &env_guard_));
-    if (test_env_uri) {
+    if (env_ == Env::Default() && getenv("MEM_ENV")) {
-      Status s = Env::LoadEnv(test_env_uri, &env_, &env_guard_);
-      EXPECT_OK(s);
-    } else if (getenv("MEM_ENV")) {
       env_guard_.reset(NewMemEnv(Env::Default()));
       env_ = env_guard_.get();
-    } else {
-      env_ = Env::Default();
     }
     EXPECT_NE(nullptr, env_);
 

db_stress_tool/db_stress_tool.cc

@@ -23,6 +23,7 @@
 #ifdef GFLAGS
 #include "db_stress_tool/db_stress_common.h"
 #include "db_stress_tool/db_stress_driver.h"
+#include "rocksdb/convenience.h"
 #ifndef NDEBUG
 #include "utilities/fault_injection_fs.h"
 #endif
@@ -34,11 +35,6 @@ static std::shared_ptr<ROCKSDB_NAMESPACE::DbStressEnvWrapper> env_wrapper_guard;
 static std::shared_ptr<CompositeEnvWrapper> fault_env_guard;
 }  // namespace
 
-static Env* GetCompositeEnv(std::shared_ptr<FileSystem> fs) {
-  static std::shared_ptr<Env> composite_env = NewCompositeEnv(fs);
-  return composite_env.get();
-}
-
 KeyGenContext key_gen_ctx;
 
 int db_stress_tool(int argc, char** argv) {
@@ -78,22 +74,14 @@ int db_stress_tool(int argc, char** argv) {
 
   if (!FLAGS_hdfs.empty()) {
     raw_env = new ROCKSDB_NAMESPACE::HdfsEnv(FLAGS_hdfs);
-  } else if (!FLAGS_env_uri.empty()) {
+  } else {
-    Status s = Env::LoadEnv(FLAGS_env_uri, &raw_env, &env_guard);
+    Status s = Env::CreateFromUri(ConfigOptions(), FLAGS_env_uri, FLAGS_fs_uri,
-    if (raw_env == nullptr) {
+                                  &raw_env, &env_guard);
-      fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str());
-      exit(1);
-    }
-  } else if (!FLAGS_fs_uri.empty()) {
-    std::shared_ptr<FileSystem> fs;
-    Status s = FileSystem::Load(FLAGS_fs_uri, &fs);
     if (!s.ok()) {
-      fprintf(stderr, "Error: %s\n", s.ToString().c_str());
+      fprintf(stderr, "Error Creating Env URI: %s: %s\n", FLAGS_env_uri.c_str(),
+              s.ToString().c_str());
       exit(1);
     }
-    raw_env = GetCompositeEnv(fs);
-  } else {
-    raw_env = Env::Default();
   }
 
 #ifndef NDEBUG

env/env.cc

@@ -16,6 +16,7 @@
 #include "memory/arena.h"
 #include "options/db_options.h"
 #include "port/port.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/options.h"
 #include "rocksdb/system_clock.h"
 #include "rocksdb/utilities/object_registry.h"
@@ -582,11 +583,18 @@ Status Env::NewLogger(const std::string& fname,
 }
 
 Status Env::LoadEnv(const std::string& value, Env** result) {
+  return CreateFromString(ConfigOptions(), value, result);
+}
+
+Status Env::CreateFromString(const ConfigOptions& config_options,
+                             const std::string& value, Env** result) {
   Env* env = *result;
   Status s;
 #ifndef ROCKSDB_LITE
+  (void)config_options;
   s = ObjectRegistry::NewInstance()->NewStaticObject<Env>(value, &env);
 #else
+  (void)config_options;
   s = Status::NotSupported("Cannot load environment in LITE mode", value);
 #endif
   if (s.ok()) {
@@ -597,18 +605,29 @@ Status Env::LoadEnv(const std::string& value, Env** result) {
 
 Status Env::LoadEnv(const std::string& value, Env** result,
                     std::shared_ptr<Env>* guard) {
+  return CreateFromString(ConfigOptions(), value, result, guard);
+}
+
+Status Env::CreateFromString(const ConfigOptions& config_options,
+                             const std::string& value, Env** result,
+                             std::shared_ptr<Env>* guard) {
   assert(result);
+  if (value.empty()) {
+    *result = Env::Default();
+    return Status::OK();
+  }
   Status s;
 #ifndef ROCKSDB_LITE
   Env* env = nullptr;
   std::unique_ptr<Env> uniq_guard;
   std::string err_msg;
   assert(guard != nullptr);
+  (void)config_options;
   env = ObjectRegistry::NewInstance()->NewObject<Env>(value, &uniq_guard,
                                                       &err_msg);
   if (!env) {
-    s = Status::NotFound(std::string("Cannot load ") + Env::Type() + ": " +
+    s = Status::NotSupported(std::string("Cannot load ") + Env::Type() + ": " +
-                         value);
+                             value);
     env = Env::Default();
   }
   if (s.ok() && uniq_guard) {
@@ -618,6 +637,7 @@ Status Env::LoadEnv(const std::string& value, Env** result,
     *result = env;
   }
 #else
+  (void)config_options;
   (void)result;
   (void)guard;
   s = Status::NotSupported("Cannot load environment in LITE mode", value);
@@ -625,6 +645,30 @@ Status Env::LoadEnv(const std::string& value, Env** result,
   return s;
 }
 
+Status Env::CreateFromUri(const ConfigOptions& config_options,
+                          const std::string& env_uri, const std::string& fs_uri,
+                          Env** result, std::shared_ptr<Env>* guard) {
+  *result = config_options.env;
+  if (env_uri.empty() && fs_uri.empty()) {
+    // Neither specified.  Use the default
+    guard->reset();
+    return Status::OK();
+  } else if (!env_uri.empty() && !fs_uri.empty()) {
+    // Both specified.  Cannot choose.  Return Invalid
+    return Status::InvalidArgument("cannot specify both fs_uri and env_uri");
+  } else if (fs_uri.empty()) {  // Only have an ENV URI.  Create an Env from it
+    return CreateFromString(config_options, env_uri, result, guard);
+  } else {
+    std::shared_ptr<FileSystem> fs;
+    Status s = FileSystem::CreateFromString(config_options, fs_uri, &fs);
+    if (s.ok()) {
+      guard->reset(new CompositeEnvWrapper(*result, fs));
+      *result = guard->get();
+    }
+    return s;
+  }
+}
+
 std::string Env::PriorityToString(Env::Priority priority) {
   switch (priority) {
     case Env::Priority::BOTTOM:

env/env_basic_test.cc

@@ -75,19 +75,30 @@ namespace {
 // The purpose of returning an empty vector (instead of nullptr) is that gtest
 // ValuesIn() will skip running tests when given an empty collection.
 std::vector<Env*> GetCustomEnvs() {
-  static Env* custom_env;
   static bool init = false;
+  static std::vector<Env*> res;
   if (!init) {
     init = true;
     const char* uri = getenv("TEST_ENV_URI");
     if (uri != nullptr) {
-      Env::LoadEnv(uri, &custom_env);
+      static std::shared_ptr<Env> env_guard;
+      static Env* custom_env;
+      Status s =
+          Env::CreateFromUri(ConfigOptions(), uri, "", &custom_env, &env_guard);
+      if (s.ok()) {
+        res.emplace_back(custom_env);
+      }
+    }
+    uri = getenv("TEST_FS_URI");
+    if (uri != nullptr) {
+      static std::shared_ptr<Env> fs_env_guard;
+      static Env* fs_env;
+      Status s =
+          Env::CreateFromUri(ConfigOptions(), "", uri, &fs_env, &fs_env_guard);
+      if (s.ok()) {
+        res.emplace_back(fs_env);
+      }
     }
-  }
-
-  std::vector<Env*> res;
-  if (custom_env != nullptr) {
-    res.emplace_back(custom_env);
   }
   return res;
 }

env/file_system.cc

@@ -3,9 +3,11 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 //
-#include "env/composite_env_wrapper.h"
 #include "rocksdb/file_system.h"
+
+#include "env/composite_env_wrapper.h"
 #include "options/db_options.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/utilities/object_registry.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -16,10 +18,18 @@ FileSystem::~FileSystem() {}
 
 Status FileSystem::Load(const std::string& value,
                         std::shared_ptr<FileSystem>* result) {
+  return CreateFromString(ConfigOptions(), value, result);
+}
+
+Status FileSystem::CreateFromString(const ConfigOptions& config_options,
+                                    const std::string& value,
+                                    std::shared_ptr<FileSystem>* result) {
   Status s;
 #ifndef ROCKSDB_LITE
+  (void)config_options;
   s = ObjectRegistry::NewInstance()->NewSharedObject<FileSystem>(value, result);
 #else
+  (void)config_options;
   (void)result;
   s = Status::NotSupported("Cannot load FileSystem in LITE mode", value);
 #endif

include/rocksdb/env.h

@@ -64,6 +64,7 @@ class ThreadStatusUpdater;
 struct ThreadStatus;
 class FileSystem;
 class SystemClock;
+struct ConfigOptions;
 
 const size_t kDefaultPageSize = 4 * 1024;
 
@@ -169,12 +170,44 @@ class Env {
   static const char* Type() { return "Environment"; }
 
   // Loads the environment specified by the input value into the result
+  // The CreateFromString alternative should be used; this method may be
+  // deprecated in a future release.
   static Status LoadEnv(const std::string& value, Env** result);
 
   // Loads the environment specified by the input value into the result
+  // The CreateFromString alternative should be used; this method may be
+  // deprecated in a future release.
   static Status LoadEnv(const std::string& value, Env** result,
                         std::shared_ptr<Env>* guard);
 
+  // Loads the environment specified by the input value into the result
+  // @see Customizable for a more detailed description of the parameters and
+  // return codes
+  //
+  // @param config_options Controls how the environment is loaded.
+  // @param value the name and associated properties for the environment.
+  // @param result On success, the environment that was loaded.
+  // @param guard If specified and the loaded environment is not static,
+  //      this value will contain the loaded environment (guard.get() ==
+  //      result).
+  // @return OK If the environment was successfully loaded (and optionally
+  // prepared)
+  // @return not-OK if the load failed.
+  static Status CreateFromString(const ConfigOptions& config_options,
+                                 const std::string& value, Env** result);
+  static Status CreateFromString(const ConfigOptions& config_options,
+                                 const std::string& value, Env** result,
+                                 std::shared_ptr<Env>* guard);
+
+  // Loads the environment specified by the env and fs uri.
+  // If both are specified, an error is returned.
+  // Otherwise, the environment is created by loading (via CreateFromString)
+  // the appropriate env/fs from the corresponding values.
+  static Status CreateFromUri(const ConfigOptions& options,
+                              const std::string& env_uri,
+                              const std::string& fs_uri, Env** result,
+                              std::shared_ptr<Env>* guard);
+
   // Return a default environment suitable for the current operating
   // system.  Sophisticated users may wish to provide their own Env
   // implementation instead of relying on this default environment.

include/rocksdb/file_system.h

@@ -46,6 +46,7 @@ class Slice;
 struct ImmutableDBOptions;
 struct MutableDBOptions;
 class RateLimiter;
+struct ConfigOptions;
 
 using AccessPattern = RandomAccessFile::AccessPattern;
 using FileAttributes = Env::FileAttributes;
@@ -209,9 +210,24 @@ class FileSystem {
   static const char* Type() { return "FileSystem"; }
 
   // Loads the FileSystem specified by the input value into the result
+  // The CreateFromString alternative should be used; this method may be
+  // deprecated in a future release.
   static Status Load(const std::string& value,
                      std::shared_ptr<FileSystem>* result);
 
+  // Loads the FileSystem specified by the input value into the result
+  // @see Customizable for a more detailed description of the parameters and
+  // return codes
+  // @param config_options Controls how the FileSystem is loaded
+  // @param value The name and optional properties describing the file system
+  //      to load.
+  // @param result On success, returns the loaded FileSystem
+  // @return OK if the FileSystem was successfully loaded.
+  // @return not-OK if the load failed.
+  static Status CreateFromString(const ConfigOptions& options,
+                                 const std::string& value,
+                                 std::shared_ptr<FileSystem>* result);
+
   // Return a default fie_system suitable for the current operating
   // system.  Sophisticated users may wish to provide their own Env
   // implementation instead of relying on this default file_system

options/db_options.cc

@@ -418,11 +418,12 @@ static std::unordered_map<std::string, OptionTypeInfo>
           OptionVerificationType::kNormal,
           (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever),
           // Parse the input value as an Env
-          [](const ConfigOptions& /*opts*/, const std::string& /*name*/,
+          [](const ConfigOptions& opts, const std::string& /*name*/,
              const std::string& value, void* addr) {
             auto old_env = static_cast<Env**>(addr);       // Get the old value
             Env* new_env = *old_env;                       // Set new to old
-            Status s = Env::LoadEnv(value, &new_env);      // Update new value
+            Status s = Env::CreateFromString(opts, value,
+                                             &new_env);    // Update new value
             if (s.ok()) {                                  // It worked
               *old_env = new_env;                          // Update the old one
             }

table/sst_file_reader_test.cc

@@ -5,11 +5,13 @@
 
 #ifndef ROCKSDB_LITE
 
+#include "rocksdb/sst_file_reader.h"
+
 #include <cinttypes>
 
 #include "port/stack_trace.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
-#include "rocksdb/sst_file_reader.h"
 #include "rocksdb/sst_file_writer.h"
 #include "table/sst_file_writer_collectors.h"
 #include "test_util/testharness.h"
@@ -37,14 +39,8 @@ class SstFileReaderTest : public testing::Test {
     sst_name_ = test::PerThreadDBPath("sst_file");
 
     Env* base_env = Env::Default();
-    const char* test_env_uri = getenv("TEST_ENV_URI");
+    EXPECT_OK(
-    if(test_env_uri) {
+        test::CreateEnvFromSystem(ConfigOptions(), &base_env, &env_guard_));
-      Env* test_env = nullptr;
-      Status s = Env::LoadEnv(test_env_uri, &test_env, &env_guard_);
-      base_env = test_env;
-      EXPECT_OK(s);
-      EXPECT_NE(Env::Default(), base_env);
-    }
     EXPECT_NE(nullptr, base_env);
     env_ = base_env;
     options_.env = env_;

test_util/testutil.cc

@@ -585,5 +585,21 @@ void DeleteDir(Env* env, const std::string& dirname) {
   TryDeleteDir(env, dirname).PermitUncheckedError();
 }
 
+Status CreateEnvFromSystem(const ConfigOptions& config_options, Env** result,
+                           std::shared_ptr<Env>* guard) {
+  const char* env_uri = getenv("TEST_ENV_URI");
+  const char* fs_uri = getenv("TEST_FS_URI");
+  if (env_uri || fs_uri) {
+    return Env::CreateFromUri(config_options,
+                              (env_uri != nullptr) ? env_uri : "",
+                              (fs_uri != nullptr) ? fs_uri : "", result, guard);
+  } else {
+    // Neither specified.  Use the default
+    *result = config_options.env;
+    guard->reset();
+    return Status::OK();
+  }
+}
+
 }  // namespace test
 }  // namespace ROCKSDB_NAMESPACE

test_util/testutil.h

@@ -865,5 +865,9 @@ Status TryDeleteDir(Env* env, const std::string& dirname);
 // Delete a directory if it exists
 void DeleteDir(Env* env, const std::string& dirname);
 
+// Creates an Env from the system environment by looking at the system
+// environment variables.
+Status CreateEnvFromSystem(const ConfigOptions& options, Env** result,
+                           std::shared_ptr<Env>* guard);
 }  // namespace test
 }  // namespace ROCKSDB_NAMESPACE

tools/db_bench_tool.cc

@@ -45,6 +45,7 @@
 #include "port/port.h"
 #include "port/stack_trace.h"
 #include "rocksdb/cache.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/filter_policy.h"
@@ -1068,15 +1069,6 @@ DEFINE_int32(thread_status_per_interval, 0,
 DEFINE_int32(perf_level, ROCKSDB_NAMESPACE::PerfLevel::kDisable,
              "Level of perf collection");
 
-#ifndef ROCKSDB_LITE
-static ROCKSDB_NAMESPACE::Env* GetCompositeEnv(
-    std::shared_ptr<ROCKSDB_NAMESPACE::FileSystem> fs) {
-  static std::shared_ptr<ROCKSDB_NAMESPACE::Env> composite_env =
-      ROCKSDB_NAMESPACE::NewCompositeEnv(fs);
-  return composite_env.get();
-}
-#endif
-
 static bool ValidateRateLimit(const char* flagname, double value) {
   const double EPSILON = 1e-10;
   if ( value < -EPSILON ) {
@@ -7693,23 +7685,20 @@ int db_bench_tool(int argc, char** argv) {
     exit(1);
   }
 
-  if (!FLAGS_env_uri.empty()) {
+  if (env_opts == 1) {
-    Status s = Env::LoadEnv(FLAGS_env_uri, &FLAGS_env, &env_guard);
+    Status s = Env::CreateFromUri(ConfigOptions(), FLAGS_env_uri, FLAGS_fs_uri,
-    if (FLAGS_env == nullptr) {
+                                  &FLAGS_env, &env_guard);
-      fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str());
+    if (!s.ok()) {
-      exit(1);
+      fprintf(stderr, "Failed creating env: %s\n", s.ToString().c_str());
-    }
-  } else if (!FLAGS_fs_uri.empty()) {
-    std::shared_ptr<FileSystem> fs;
-    Status s = FileSystem::Load(FLAGS_fs_uri, &fs);
-    if (fs == nullptr) {
-      fprintf(stderr, "Error: %s\n", s.ToString().c_str());
       exit(1);
     }
-    FLAGS_env = GetCompositeEnv(fs);
   } else if (FLAGS_simulate_hybrid_fs_file != "") {
-    FLAGS_env = GetCompositeEnv(std::make_shared<SimulatedHybridFileSystem>(
+    //**TODO: Make the simulate fs something that can be loaded
-        FileSystem::Default(), FLAGS_simulate_hybrid_fs_file));
+    // from the ObjectRegistry...
+    static std::shared_ptr<ROCKSDB_NAMESPACE::Env> composite_env =
+        NewCompositeEnv(std::make_shared<SimulatedHybridFileSystem>(
+            FileSystem::Default(), FLAGS_simulate_hybrid_fs_file));
+    FLAGS_env = composite_env.get();
   }
 #endif  // ROCKSDB_LITE
   if (FLAGS_use_existing_keys && !FLAGS_use_existing_db) {

tools/ldb_cmd.cc

@@ -287,16 +287,6 @@ LDBCommand* LDBCommand::SelectCommand(const ParsedParams& parsed_params) {
   return nullptr;
 }
 
-static Env* GetCompositeEnv(std::shared_ptr<FileSystem> fs) {
-  static std::shared_ptr<Env> composite_env = NewCompositeEnv(fs);
-  return composite_env.get();
-}
-
-static Env* GetCompositeBackupEnv(std::shared_ptr<FileSystem> fs) {
-  static std::shared_ptr<Env> composite_backup_env = NewCompositeEnv(fs);
-  return composite_backup_env.get();
-}
-
 /* Run the command, and return the execute result. */
 void LDBCommand::Run() {
   if (!exec_state_.IsNotStarted()) {
@@ -305,33 +295,13 @@ void LDBCommand::Run() {
 
   if (!options_.env || options_.env == Env::Default()) {
     Env* env = Env::Default();
-
+    Status s = Env::CreateFromUri(config_options_, env_uri_, fs_uri_, &env,
-    if (!env_uri_.empty() && !fs_uri_.empty()) {
+                                  &env_guard_);
-      std::string err =
+    if (!s.ok()) {
-          "Error: you may not specity both "
+      fprintf(stderr, "%s\n", s.ToString().c_str());
-          "fs_uri and fs_env.";
+      exec_state_ = LDBCommandExecuteResult::Failed(s.ToString());
-      fprintf(stderr, "%s\n", err.c_str());
-      exec_state_ = LDBCommandExecuteResult::Failed(err);
       return;
     }
-    if (!env_uri_.empty()) {
-      Status s = Env::LoadEnv(env_uri_, &env, &env_guard_);
-      if (!s.ok() && !s.IsNotFound()) {
-        fprintf(stderr, "LoadEnv: %s\n", s.ToString().c_str());
-        exec_state_ = LDBCommandExecuteResult::Failed(s.ToString());
-        return;
-      }
-    } else if (!fs_uri_.empty()) {
-      std::shared_ptr<FileSystem> fs;
-      Status s = FileSystem::Load(fs_uri_, &fs);
-      if (fs == nullptr) {
-        fprintf(stderr, "error: %s\n", s.ToString().c_str());
-        exec_state_ = LDBCommandExecuteResult::Failed(s.ToString());
-        return;
-      }
-      env = GetCompositeEnv(fs);
-    }
-
     options_.env = env;
   }
 
@@ -3218,19 +3188,17 @@ void BackupCommand::DoCommand() {
   }
   fprintf(stdout, "open db OK\n");
 
-  Env* custom_env = nullptr;
+  Env* custom_env = backup_env_guard_.get();
-  if (!backup_fs_uri_.empty()) {
+  if (custom_env == nullptr) {
-    std::shared_ptr<FileSystem> fs;
+    Status s =
-    Status s = FileSystem::Load(backup_fs_uri_, &fs);
+        Env::CreateFromUri(config_options_, backup_env_uri_, backup_fs_uri_,
-    if (fs == nullptr) {
+                           &custom_env, &backup_env_guard_);
+    if (!s.ok()) {
       exec_state_ = LDBCommandExecuteResult::Failed(s.ToString());
       return;
     }
-    custom_env = GetCompositeBackupEnv(fs);
-  } else {
-    Env::LoadEnv(backup_env_uri_, &custom_env, &backup_env_guard_);
-    assert(custom_env != nullptr);
   }
+  assert(custom_env != nullptr);
 
   BackupableDBOptions backup_options =
       BackupableDBOptions(backup_dir_, custom_env);
@@ -3265,19 +3233,17 @@ void RestoreCommand::Help(std::string& ret) {
 }
 
 void RestoreCommand::DoCommand() {
-  Env* custom_env = nullptr;
+  Env* custom_env = backup_env_guard_.get();
-  if (!backup_fs_uri_.empty()) {
+  if (custom_env == nullptr) {
-    std::shared_ptr<FileSystem> fs;
+    Status s =
-    Status s = FileSystem::Load(backup_fs_uri_, &fs);
+        Env::CreateFromUri(config_options_, backup_env_uri_, backup_fs_uri_,
-    if (fs == nullptr) {
+                           &custom_env, &backup_env_guard_);
+    if (!s.ok()) {
       exec_state_ = LDBCommandExecuteResult::Failed(s.ToString());
       return;
     }
-    custom_env = GetCompositeBackupEnv(fs);
-  } else {
-    Env::LoadEnv(backup_env_uri_, &custom_env, &backup_env_guard_);
-    assert(custom_env != nullptr);
   }
+  assert(custom_env != nullptr);
 
   std::unique_ptr<BackupEngineReadOnly> restore_engine;
   Status status;

tools/ldb_cmd_test.cc

@@ -12,6 +12,7 @@
 #include "env/composite_env_wrapper.h"
 #include "file/filename.h"
 #include "port/stack_trace.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/file_checksum.h"
 #include "rocksdb/utilities/options_util.h"
 #include "test_util/sync_point.h"
@@ -31,12 +32,8 @@ class LdbCmdTest : public testing::Test {
   LdbCmdTest() : testing::Test() {}
 
   Env* TryLoadCustomOrDefaultEnv() {
-    const char* test_env_uri = getenv("TEST_ENV_URI");
-    if (!test_env_uri) {
-      return Env::Default();
-    }
     Env* env = Env::Default();
-    Env::LoadEnv(test_env_uri, &env, &env_guard_);
+    EXPECT_OK(test::CreateEnvFromSystem(ConfigOptions(), &env, &env_guard_));
     return env;
   }
 

tools/sst_dump_test.cc

@@ -10,11 +10,12 @@
 #ifndef ROCKSDB_LITE
 
 #include <stdint.h>
-#include "rocksdb/sst_dump_tool.h"
 
 #include "file/random_access_file_reader.h"
 #include "port/stack_trace.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/filter_policy.h"
+#include "rocksdb/sst_dump_tool.h"
 #include "table/block_based/block_based_table_factory.h"
 #include "table/table_builder.h"
 #include "test_util/testharness.h"
@@ -56,10 +57,7 @@ class SSTDumpToolTest : public testing::Test {
 
  public:
   SSTDumpToolTest() : env_(Env::Default()) {
-    const char* test_env_uri = getenv("TEST_ENV_URI");
+    EXPECT_OK(test::CreateEnvFromSystem(ConfigOptions(), &env_, &env_guard_));
-    if (test_env_uri) {
-      Env::LoadEnv(test_env_uri, &env_, &env_guard_);
-    }
     test_dir_ = test::PerThreadDBPath(env_, "sst_dump_test_db");
     Status s = env_->CreateDirIfMissing(test_dir_);
     EXPECT_OK(s);

tools/sst_dump_tool.cc

@@ -132,16 +132,8 @@ bool ParseIntArg(const char* arg, const std::string arg_name,
 }
 }  // namespace
 
-static ROCKSDB_NAMESPACE::Env* GetCompositeEnv(
-    std::shared_ptr<ROCKSDB_NAMESPACE::FileSystem> fs) {
-  static std::shared_ptr<ROCKSDB_NAMESPACE::Env> composite_env =
-      ROCKSDB_NAMESPACE::NewCompositeEnv(fs);
-  return composite_env.get();
-}
-
 int SSTDumpTool::Run(int argc, char const* const* argv, Options options) {
-  const char* env_uri = nullptr;
+  std::string env_uri, fs_uri;
-  const char* fs_uri = nullptr;
   const char* dir_or_file = nullptr;
   uint64_t read_num = std::numeric_limits<uint64_t>::max();
   std::string command;
@@ -354,32 +346,17 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) {
   // If caller of SSTDumpTool::Run(...) does not specify a different env other
   // than Env::Default(), then try to load custom env based on env_uri/fs_uri.
   // Otherwise, the caller is responsible for creating custom env.
-
+  {
-  if (env_uri && fs_uri) {
+    ConfigOptions config_options;
-    fprintf(stderr, "cannot specify --fs_uri and --env_uri.\n\n");
+    config_options.env = options.env;
-    exit(1);
+    Status s = Env::CreateFromUri(config_options, env_uri, fs_uri, &options.env,
-  }
+                                  &env_guard);
-
+    if (!s.ok()) {
-  if (!options.env || options.env == ROCKSDB_NAMESPACE::Env::Default()) {
+      fprintf(stderr, "CreateEnvFromUri: %s\n", s.ToString().c_str());
-    Env* env = Env::Default();
+      exit(1);
-    if (env_uri) {
+    } else {
-      Status s = Env::LoadEnv(env_uri ? env_uri : "", &env, &env_guard);
+      fprintf(stdout, "options.env is %p\n", options.env);
-      if (!s.ok() && !s.IsNotFound()) {
-        fprintf(stderr, "LoadEnv: %s\n", s.ToString().c_str());
-        exit(1);
-      }
-    } else if (fs_uri) {
-      std::shared_ptr<FileSystem> fs;
-      Status s = FileSystem::Load(fs_uri, &fs);
-      if (fs == nullptr) {
-        fprintf(stderr, "FileSystem Load: %s\n", s.ToString().c_str());
-        exit(1);
-      }
-      env = GetCompositeEnv(fs);
     }
-    options.env = env;
-  } else {
-    fprintf(stdout, "options.env is %p\n", options.env);
   }
 
   std::vector<std::string> filenames;