Clarify some DB::Open,OpenForReadOnly semantics (#8379)

Summary:
Longstanding tech debt

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8379

Test Plan: Better than not having an API contract

Reviewed By: jay-zhuang

Differential Revision: D29011131

Pulled By: pdillinger

fbshipit-source-id: 2c4796177733651954024fc17875f8642ca08d09

cache/cache_entry_roles.h

@@ -27,7 +27,7 @@ enum class CacheEntryRole {
   kIndexBlock,
   // Other kinds of block-based table block
   kOtherBlock,
-  // WriteBufferManager resevations to account for memtable usage
+  // WriteBufferManager reservations to account for memtable usage
   kWriteBuffer,
   // Default bucket, for miscellaneous cache entries. Do not use for
   // entries that could potentially add up to large usage.

include/rocksdb/db.h

@@ -140,11 +140,15 @@ typedef std::unordered_map<std::string, std::shared_ptr<const TableProperties>>
 // and a number of wrapper implementations.
 class DB {
  public:
-  // Open the database with the specified "name".
+  // Open the database with the specified "name" for reads and writes.
   // Stores a pointer to a heap-allocated database in *dbptr and returns
   // OK on success.
-  // Stores nullptr in *dbptr and returns a non-OK status on error.
-  // Caller should delete *dbptr when it is no longer needed.
+  // Stores nullptr in *dbptr and returns a non-OK status on error, including
+  // if the DB is already open (read-write) by another DB object. (This
+  // guarantee depends on options.env->LockFile(), which might not provide
+  // this guarantee in a custom Env implementation.)
+  //
+  // Caller must delete *dbptr when it is no longer needed.
   static Status Open(const Options& options, const std::string& name,
                      DB** dbptr);
 
@@ -153,6 +157,12 @@ class DB {
   // If the db is opened in read only mode, then no compactions
   // will happen.
   //
+  // While a given DB can be simultaneously open via OpenForReadOnly
+  // by any number of readers, if a DB is simultaneously open by Open
+  // and OpenForReadOnly, the read-only instance has undefined behavior
+  // (though can often succeed if quickly closed) and the read-write
+  // instance is unaffected. See also OpenAsSecondary.
+  //
   // Not supported in ROCKSDB_LITE, in which case the function will
   // return Status::NotSupported.
   static Status OpenForReadOnly(const Options& options, const std::string& name,
@@ -165,6 +175,12 @@ class DB {
   // column family. The default column family name is 'default' and it's stored
   // in ROCKSDB_NAMESPACE::kDefaultColumnFamilyName
   //
+  // While a given DB can be simultaneously open via OpenForReadOnly
+  // by any number of readers, if a DB is simultaneously open by Open
+  // and OpenForReadOnly, the read-only instance has undefined behavior
+  // (though can often succeed if quickly closed) and the read-write
+  // instance is unaffected. See also OpenAsSecondary.
+  //
   // Not supported in ROCKSDB_LITE, in which case the function will
   // return Status::NotSupported.
   static Status OpenForReadOnly(