diff --git a/CMakeLists.txt b/CMakeLists.txt index d7ab5b488..63217b888 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -713,6 +713,7 @@ set(SOURCES env/file_system_tracer.cc env/fs_remap.cc env/mock_env.cc + env/unique_id.cc file/delete_scheduler.cc file/file_prefetch_buffer.cc file/file_util.cc diff --git a/HISTORY.md b/HISTORY.md index d2298059c..238a4ac7c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -3,6 +3,7 @@ ### Bug Fixes * Allow secondary instance to refresh iterator. Assign read seq after referencing SuperVersion. * Fixed a bug of secondary instance's last_sequence going backward, and reads on the secondary fail to see recent updates from the primary. +* Fixed a bug that could lead to duplicate DB ID or DB session ID in POSIX environments without /proc/sys/kernel/random/uuid. * Fix a race in DumpStats() with column family destruction due to not taking a Ref on each entry while iterating the ColumnFamilySet. ### New Features diff --git a/TARGETS b/TARGETS index ac10bcacb..214e35e45 100644 --- a/TARGETS +++ b/TARGETS @@ -225,6 +225,7 @@ cpp_library( "env/fs_remap.cc", "env/io_posix.cc", "env/mock_env.cc", + "env/unique_id.cc", "file/delete_scheduler.cc", "file/file_prefetch_buffer.cc", "file/file_util.cc", @@ -545,6 +546,7 @@ cpp_library( "env/fs_remap.cc", "env/io_posix.cc", "env/mock_env.cc", + "env/unique_id.cc", "file/delete_scheduler.cc", "file/file_prefetch_buffer.cc", "file/file_util.cc", diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index 228b24c24..2d020fe6c 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -53,6 +53,7 @@ #include "db/version_set.h" #include "db/write_batch_internal.h" #include "db/write_callback.h" +#include "env/unique_id.h" #include "file/file_util.h" #include "file/filename.h" #include "file/random_access_file_reader.h" @@ -3938,7 +3939,8 @@ Status DBImpl::GetDbIdentityFromIdentityFile(std::string* identity) const { return s; } - // If last character is '\n' remove it from identity + // If last character is '\n' remove it from identity. (Old implementations + // of Env::GenerateUniqueId() would include a trailing '\n'.) if (identity->size() > 0 && identity->back() == '\n') { identity->pop_back(); } @@ -3950,10 +3952,11 @@ Status DBImpl::GetDbSessionId(std::string& session_id) const { return Status::OK(); } -std::string DBImpl::GenerateDbSessionId(Env* env) { - // GenerateUniqueId() generates an identifier that has a negligible - // probability of being duplicated, ~128 bits of entropy - std::string uuid = env->GenerateUniqueId(); +std::string DBImpl::GenerateDbSessionId(Env*) { + // GenerateRawUniqueId() generates an identifier that has a negligible + // probability of being duplicated. It should have full 128 bits of entropy. + uint64_t a, b; + GenerateRawUniqueId(&a, &b); // Hash and reformat that down to a more compact format, 20 characters // in base-36 ([0-9A-Z]), which is ~103 bits of entropy, which is enough @@ -3962,18 +3965,10 @@ std::string DBImpl::GenerateDbSessionId(Env* env) { // * Save ~ dozen bytes per SST file // * Shorter shared backup file names (some platforms have low limits) // * Visually distinct from DB id format - uint64_t a = NPHash64(uuid.data(), uuid.size(), 1234U); - uint64_t b = NPHash64(uuid.data(), uuid.size(), 5678U); - std::string db_session_id; - db_session_id.resize(20); - static const char* const base36 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - size_t i = 0; - for (; i < 10U; ++i, a /= 36U) { - db_session_id[i] = base36[a % 36]; - } - for (; i < 20U; ++i, b /= 36U) { - db_session_id[i] = base36[b % 36]; - } + std::string db_session_id(20U, '\0'); + char* buf = &db_session_id[0]; + PutBaseChars<36>(&buf, 10, a, /*uppercase*/ true); + PutBaseChars<36>(&buf, 10, b, /*uppercase*/ true); return db_session_id; } diff --git a/db/merge_test.cc b/db/merge_test.cc index 2cca0735e..ba25267ac 100644 --- a/db/merge_test.cc +++ b/db/merge_test.cc @@ -117,6 +117,9 @@ std::shared_ptr OpenDb(const std::string& dbname, const bool ttl = false, #endif // !ROCKSDB_LITE EXPECT_OK(s); assert(s.ok()); + // Allowed to call NowNanos during DB creation (in GenerateRawUniqueId() for + // session ID) + EnvMergeTest::now_nanos_count_ = 0; return std::shared_ptr(db); } @@ -463,6 +466,8 @@ void testPartialMerge(Counters* counters, DB* db, size_t max_merge, ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); ASSERT_EQ(tmp_sum, counters->assert_get("c")); ASSERT_EQ(num_partial_merge_calls, 0U); + // NowNanos was previously called in MergeHelper::FilterMerge(), which + // harmed performance. ASSERT_EQ(EnvMergeTest::now_nanos_count_, 0U); } diff --git a/env/env.cc b/env/env.cc index 77349fa68..5aa25e523 100644 --- a/env/env.cc +++ b/env/env.cc @@ -12,6 +12,7 @@ #include #include "env/composite_env_wrapper.h" +#include "env/unique_id.h" #include "logging/env_logger.h" #include "memory/arena.h" #include "options/db_options.h" @@ -21,6 +22,7 @@ #include "rocksdb/system_clock.h" #include "rocksdb/utilities/object_registry.h" #include "util/autovector.h" +#include "util/string_util.h" namespace ROCKSDB_NAMESPACE { namespace { @@ -737,6 +739,46 @@ Status Env::GetHostNameString(std::string* result) { return s; } +std::string Env::GenerateUniqueId() { + std::string result; + bool success = port::GenerateRfcUuid(&result); + if (!success) { + // Fall back on our own way of generating a unique ID and adapt it to + // RFC 4122 variant 1 version 4 (a random ID). + // https://en.wikipedia.org/wiki/Universally_unique_identifier + // We already tried GenerateRfcUuid so no need to try it again in + // GenerateRawUniqueId + constexpr bool exclude_port_uuid = true; + uint64_t upper, lower; + GenerateRawUniqueId(&upper, &lower, exclude_port_uuid); + + // Set 4-bit version to 4 + upper = (upper & (~uint64_t{0xf000})) | 0x4000; + // Set unary-encoded variant to 1 (0b10) + lower = (lower & (~(uint64_t{3} << 62))) | (uint64_t{2} << 62); + + // Use 36 character format of RFC 4122 + result.resize(36U); + char* buf = &result[0]; + PutBaseChars<16>(&buf, 8, upper >> 32, /*!uppercase*/ false); + *(buf++) = '-'; + PutBaseChars<16>(&buf, 4, upper >> 16, /*!uppercase*/ false); + *(buf++) = '-'; + PutBaseChars<16>(&buf, 4, upper, /*!uppercase*/ false); + *(buf++) = '-'; + PutBaseChars<16>(&buf, 4, lower >> 48, /*!uppercase*/ false); + *(buf++) = '-'; + PutBaseChars<16>(&buf, 12, lower, /*!uppercase*/ false); + assert(buf == &result[36]); + + // Verify variant 1 version 4 + assert(result[14] == '4'); + assert(result[19] == '8' || result[19] == '9' || result[19] == 'a' || + result[19] == 'b'); + } + return result; +} + SequentialFile::~SequentialFile() { } diff --git a/env/env_posix.cc b/env/env_posix.cc index 583c024de..e6d657764 100644 --- a/env/env_posix.cc +++ b/env/env_posix.cc @@ -469,32 +469,6 @@ void PosixEnv::WaitForJoin() { } // namespace -std::string Env::GenerateUniqueId() { - std::string uuid_file = "/proc/sys/kernel/random/uuid"; - std::shared_ptr fs = FileSystem::Default(); - - Status s = fs->FileExists(uuid_file, IOOptions(), nullptr); - if (s.ok()) { - std::string uuid; - s = ReadFileToString(fs.get(), uuid_file, &uuid); - if (s.ok()) { - return uuid; - } - } - // Could not read uuid_file - generate uuid using "nanos-random" - Random64 r(time(nullptr)); - uint64_t random_uuid_portion = - r.Uniform(std::numeric_limits::max()); - uint64_t nanos_uuid_portion = NowNanos(); - char uuid2[200]; - snprintf(uuid2, - 200, - "%lx-%lx", - (unsigned long)nanos_uuid_portion, - (unsigned long)random_uuid_portion); - return uuid2; -} - // // Default Posix Env // diff --git a/env/env_test.cc b/env/env_test.cc index ac128b6fc..d8403d3d3 100644 --- a/env/env_test.cc +++ b/env/env_test.cc @@ -18,10 +18,10 @@ #include -#include -#include #include #include +#include +#include #ifdef OS_LINUX #include @@ -35,8 +35,10 @@ #include #endif +#include "db/db_impl/db_impl.h" #include "env/env_chroot.h" #include "env/env_encryption_ctr.h" +#include "env/unique_id.h" #include "logging/log_buffer.h" #include "port/malloc.h" #include "port/port.h" @@ -2458,6 +2460,190 @@ TEST_F(EncryptionProviderTest, LoadROT13Cipher) { #endif // ROCKSDB_LITE +namespace { + +constexpr size_t kThreads = 8; +constexpr size_t kIdsPerThread = 1000; + +// This is a mini-stress test to check for duplicates in functions like +// GenerateUniqueId() +template > +struct NoDuplicateMiniStressTest { + std::unordered_set ids; + std::mutex mutex; + Env* env; + + NoDuplicateMiniStressTest() { env = Env::Default(); } + + virtual ~NoDuplicateMiniStressTest() {} + + void Run() { + std::array threads; + for (size_t i = 0; i < kThreads; ++i) { + threads[i] = std::thread([&]() { ThreadFn(); }); + } + for (auto& thread : threads) { + thread.join(); + } + // All must be unique + ASSERT_EQ(ids.size(), kThreads * kIdsPerThread); + } + + void ThreadFn() { + std::array my_ids; + // Generate in parallel threads as fast as possible + for (size_t i = 0; i < kIdsPerThread; ++i) { + my_ids[i] = Generate(); + } + // Now collate + std::lock_guard lock(mutex); + for (auto& id : my_ids) { + ids.insert(id); + } + } + + virtual IdType Generate() = 0; +}; + +void VerifyRfcUuids(const std::unordered_set& uuids) { + if (uuids.empty()) { + return; + } +} + +using uint64_pair_t = std::pair; +struct HashUint64Pair { + std::size_t operator()( + std::pair const& u) const noexcept { + // Assume suitable distribution already + return static_cast(u.first ^ u.second); + } +}; + +} // namespace + +TEST_F(EnvTest, GenerateUniqueId) { + struct MyStressTest : public NoDuplicateMiniStressTest { + std::string Generate() override { return env->GenerateUniqueId(); } + }; + + MyStressTest t; + t.Run(); + + // Basically verify RFC-4122 format + for (auto& uuid : t.ids) { + ASSERT_EQ(36U, uuid.size()); + ASSERT_EQ('-', uuid[8]); + ASSERT_EQ('-', uuid[13]); + ASSERT_EQ('-', uuid[18]); + ASSERT_EQ('-', uuid[23]); + } +} + +TEST_F(EnvTest, GenerateDbSessionId) { + struct MyStressTest : public NoDuplicateMiniStressTest { + std::string Generate() override { return DBImpl::GenerateDbSessionId(env); } + }; + + MyStressTest t; + t.Run(); + + // Basically verify session ID + for (auto& id : t.ids) { + ASSERT_EQ(20U, id.size()); + } +} + +constexpr bool kRequirePortGenerateRfcUuid = +#if defined(OS_LINUX) || defined(OS_ANDROID) || defined(OS_WIN) + true; +#else + false; +#endif + +TEST_F(EnvTest, PortGenerateRfcUuid) { + if (!kRequirePortGenerateRfcUuid) { + ROCKSDB_GTEST_SKIP("Not supported/expected on this platform"); + return; + } + struct MyStressTest : public NoDuplicateMiniStressTest { + std::string Generate() override { + std::string u; + assert(port::GenerateRfcUuid(&u)); + return u; + } + }; + + MyStressTest t; + t.Run(); + + // Extra verification on versions and variants + VerifyRfcUuids(t.ids); +} + +// Test the atomic, linear generation of GenerateRawUuid +TEST_F(EnvTest, GenerateRawUniqueId) { + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + GenerateRawUniqueId(&p.first, &p.second); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + +// Test that each entropy source ("track") is at least adequate +TEST_F(EnvTest, GenerateRawUniqueIdTrackPortUuidOnly) { + if (!kRequirePortGenerateRfcUuid) { + ROCKSDB_GTEST_SKIP("Not supported/expected on this platform"); + return; + } + + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + TEST_GenerateRawUniqueId(&p.first, &p.second, false, true, true); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + +TEST_F(EnvTest, GenerateRawUniqueIdTrackEnvDetailsOnly) { + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + TEST_GenerateRawUniqueId(&p.first, &p.second, true, false, true); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + +TEST_F(EnvTest, GenerateRawUniqueIdTrackRandomDeviceOnly) { + struct MyStressTest + : public NoDuplicateMiniStressTest { + uint64_pair_t Generate() override { + uint64_pair_t p; + TEST_GenerateRawUniqueId(&p.first, &p.second, true, true, false); + return p; + } + }; + + MyStressTest t; + t.Run(); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/env/unique_id.cc b/env/unique_id.cc new file mode 100644 index 000000000..ab8216a2b --- /dev/null +++ b/env/unique_id.cc @@ -0,0 +1,144 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "env/unique_id.h" + +#include +#include +#include +#include + +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/version.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +struct GenerateRawUniqueIdOpts { + Env* env = Env::Default(); + bool exclude_port_uuid = false; + bool exclude_env_details = false; + bool exclude_random_device = false; +}; + +// Each of these "tracks" below should be sufficient for generating 128 bits +// of entropy, after hashing the raw bytes. The tracks are separable for +// testing purposes, but in production we combine as many tracks as possible +// to ensure quality results even if some environments have degraded +// capabilities or quality in some APIs. +// +// This approach has not been validated for use in cryptography. The goal is +// generating globally unique values with high probability without coordination +// between instances. +// +// Linux performance: EntropyTrackRandomDevice is much faster than +// EntropyTrackEnvDetails, which is much faster than EntropyTrackPortUuid. + +struct EntropyTrackPortUuid { + std::array uuid; + + void Populate(const GenerateRawUniqueIdOpts& opts) { + if (opts.exclude_port_uuid) { + return; + } + std::string s; + port::GenerateRfcUuid(&s); + if (s.size() >= uuid.size()) { + std::copy_n(s.begin(), uuid.size(), uuid.begin()); + } + } +}; + +struct EntropyTrackEnvDetails { + std::array hostname_buf; + int64_t process_id; + uint64_t thread_id; + int64_t unix_time; + uint64_t nano_time; + + void Populate(const GenerateRawUniqueIdOpts& opts) { + if (opts.exclude_env_details) { + return; + } + opts.env->GetHostName(hostname_buf.data(), hostname_buf.size()) + .PermitUncheckedError(); + process_id = port::GetProcessID(); + thread_id = opts.env->GetThreadID(); + opts.env->GetCurrentTime(&unix_time).PermitUncheckedError(); + nano_time = opts.env->NowNanos(); + } +}; + +struct EntropyTrackRandomDevice { + using RandType = std::random_device::result_type; + static constexpr size_t kNumRandVals = + /* generous bits */ 192U / (8U * sizeof(RandType)); + std::array rand_vals; + + void Populate(const GenerateRawUniqueIdOpts& opts) { + if (opts.exclude_random_device) { + return; + } + std::random_device r; + for (auto& val : rand_vals) { + val = r(); + } + } +}; + +struct Entropy { + uint64_t version_identifier; + EntropyTrackRandomDevice et1; + EntropyTrackEnvDetails et2; + EntropyTrackPortUuid et3; + + void Populate(const GenerateRawUniqueIdOpts& opts) { + // If we change the format of what goes into the entropy inputs, it's + // conceivable there could be a physical collision in the hash input + // even though they are logically different. This value should change + // if there's a change to the "schema" here, including byte order. + version_identifier = (uint64_t{ROCKSDB_MAJOR} << 32) + + (uint64_t{ROCKSDB_MINOR} << 16) + + uint64_t{ROCKSDB_PATCH}; + et1.Populate(opts); + et2.Populate(opts); + et3.Populate(opts); + } +}; + +void GenerateRawUniqueIdImpl(uint64_t* a, uint64_t* b, + const GenerateRawUniqueIdOpts& opts) { + Entropy e; + std::memset(&e, 0, sizeof(e)); + e.Populate(opts); + Hash2x64(reinterpret_cast(&e), sizeof(e), a, b); +} + +} // namespace + +void GenerateRawUniqueId(uint64_t* a, uint64_t* b, bool exclude_port_uuid) { + GenerateRawUniqueIdOpts opts; + opts.exclude_port_uuid = exclude_port_uuid; + assert(!opts.exclude_env_details); + assert(!opts.exclude_random_device); + GenerateRawUniqueIdImpl(a, b, opts); +} + +#ifndef NDEBUG +void TEST_GenerateRawUniqueId(uint64_t* a, uint64_t* b, bool exclude_port_uuid, + bool exclude_env_details, + bool exclude_random_device) { + GenerateRawUniqueIdOpts opts; + opts.exclude_port_uuid = exclude_port_uuid; + opts.exclude_env_details = exclude_env_details; + opts.exclude_random_device = exclude_random_device; + GenerateRawUniqueIdImpl(a, b, opts); +} +#endif + +} // namespace ROCKSDB_NAMESPACE diff --git a/env/unique_id.h b/env/unique_id.h new file mode 100644 index 000000000..890d4d776 --- /dev/null +++ b/env/unique_id.h @@ -0,0 +1,40 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// This file is for functions that extract novel entropy or sources of +// uniqueness from the execution environment. (By contrast, random.h is +// for algorithmic pseudorandomness.) +// +// These functions could eventually migrate to public APIs, such as in Env. + +#pragma once + +#include + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +// Generates a new 128-bit identifier that is universally unique +// (with high probability) for each call. The result is split into +// two 64-bit pieces. This function has NOT been validated for use in +// cryptography. +// +// This is used in generating DB session IDs and by Env::GenerateUniqueId +// (used for DB IDENTITY) if the platform does not provide a generator of +// RFC 4122 UUIDs or fails somehow. (Set exclude_port_uuid=true if this +// function is used as a fallback for GenerateRfcUuid, because no need +// trying it again.) +void GenerateRawUniqueId(uint64_t* a, uint64_t* b, + bool exclude_port_uuid = false); + +#ifndef NDEBUG +// A version of above with options for challenge testing +void TEST_GenerateRawUniqueId(uint64_t* a, uint64_t* b, bool exclude_port_uuid, + bool exclude_env_details, + bool exclude_random_device); +#endif + +} // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/env.h b/include/rocksdb/env.h index a4463060a..76a3bcf79 100644 --- a/include/rocksdb/env.h +++ b/include/rocksdb/env.h @@ -559,7 +559,10 @@ class Env { // Converts seconds-since-Jan-01-1970 to a printable string virtual std::string TimeToString(uint64_t time) = 0; - // Generates a unique id that can be used to identify a db + // Generates a human-readable unique ID that can be used to identify a DB. + // In built-in implementations, this is an RFC-4122 UUID string, but might + // not be in all implementations. Overriding is not recommended. + // NOTE: this has not be validated for use in cryptography virtual std::string GenerateUniqueId(); // OptimizeForLogWrite will create a new EnvOptions object that is a copy of diff --git a/port/port_posix.cc b/port/port_posix.cc index 57f19e33d..8615f11d6 100644 --- a/port/port_posix.cc +++ b/port/port_posix.cc @@ -25,6 +25,8 @@ #include #include +#include +#include #include "util/string_util.h" @@ -265,6 +267,18 @@ void SetCpuPriority(ThreadId id, CpuPriority priority) { int64_t GetProcessID() { return getpid(); } +bool GenerateRfcUuid(std::string* output) { + output->clear(); + std::ifstream f("/proc/sys/kernel/random/uuid"); + std::getline(f, /*&*/ *output); + if (output->size() == 36) { + return true; + } else { + output->clear(); + return false; + } +} + } // namespace port } // namespace ROCKSDB_NAMESPACE diff --git a/port/port_posix.h b/port/port_posix.h index fa23f9c33..9bea21039 100644 --- a/port/port_posix.h +++ b/port/port_posix.h @@ -221,5 +221,9 @@ extern void SetCpuPriority(ThreadId id, CpuPriority priority); int64_t GetProcessID(); +// Uses platform APIs to generate a 36-character RFC-4122 UUID. Returns +// true on success or false on failure. +bool GenerateRfcUuid(std::string* output); + } // namespace port } // namespace ROCKSDB_NAMESPACE diff --git a/port/win/env_win.cc b/port/win/env_win.cc index 93e04d389..2b141c55d 100644 --- a/port/win/env_win.cc +++ b/port/win/env_win.cc @@ -1404,25 +1404,6 @@ void WinEnv::IncBackgroundThreadsIfNeeded(int num, Env::Priority pri) { } // namespace port -std::string Env::GenerateUniqueId() { - std::string result; - - UUID uuid; - UuidCreateSequential(&uuid); - - RPC_CSTR rpc_str; - auto status = UuidToStringA(&uuid, &rpc_str); - (void)status; - assert(status == RPC_S_OK); - - result = reinterpret_cast(rpc_str); - - status = RpcStringFreeA(&rpc_str); - assert(status == RPC_S_OK); - - return result; -} - std::shared_ptr FileSystem::Default() { return port::WinFileSystem::Default(); } diff --git a/port/win/port_win.cc b/port/win/port_win.cc index b28a44895..f0984a9a2 100644 --- a/port/win/port_win.cc +++ b/port/win/port_win.cc @@ -11,18 +11,19 @@ #include "port/win/port_win.h" +#include #include -#include "port/port_dirent.h" -#include "port/sys_time.h" - -#include +#include #include -#include #include -#include -#include #include +#include +#include +#include + +#include "port/port_dirent.h" +#include "port/sys_time.h" #ifdef ROCKSDB_WINDOWS_UTF8_FILENAMES // utf8 <-> utf16 @@ -279,6 +280,25 @@ void SetCpuPriority(ThreadId id, CpuPriority priority) { int64_t GetProcessID() { return GetCurrentProcessId(); } +bool GenerateRfcUuid(std::string* output) { + UUID uuid; + UuidCreateSequential(&uuid); + + RPC_CSTR rpc_str; + auto status = UuidToStringA(&uuid, &rpc_str); + if (status != RPC_S_OK) { + return false; + } + + // rpc_str is nul-terminated + *output = reinterpret_cast(rpc_str); + + status = RpcStringFreeA(&rpc_str); + assert(status == RPC_S_OK); + + return true; +} + } // namespace port } // namespace ROCKSDB_NAMESPACE diff --git a/port/win/port_win.h b/port/win/port_win.h index 7de9b61c5..2de75179d 100644 --- a/port/win/port_win.h +++ b/port/win/port_win.h @@ -349,6 +349,10 @@ extern void SetCpuPriority(ThreadId id, CpuPriority priority); int64_t GetProcessID(); +// Uses platform APIs to generate a 36-character RFC-4122 UUID. Returns +// true on success or false on failure. +bool GenerateRfcUuid(std::string* output); + } // namespace port diff --git a/src.mk b/src.mk index 2da058495..0506beeb7 100644 --- a/src.mk +++ b/src.mk @@ -94,6 +94,7 @@ LIB_SOURCES = \ env/file_system_tracer.cc \ env/io_posix.cc \ env/mock_env.cc \ + env/unique_id.cc \ file/delete_scheduler.cc \ file/file_prefetch_buffer.cc \ file/file_util.cc \ diff --git a/util/hash.cc b/util/hash.cc index eddaa6dd2..f53aa8ff1 100644 --- a/util/hash.cc +++ b/util/hash.cc @@ -113,4 +113,11 @@ Unsigned128 Hash128(const char* data, size_t n) { return (Unsigned128{h.high64} << 64) | (h.low64); } +void Hash2x64(const char* data, size_t n, uint64_t* high64, uint64_t* low64) { + // Same as seed = 0 + auto h = XXH3_128bits(data, n); + *high64 = h.high64; + *low64 = h.low64; +} + } // namespace ROCKSDB_NAMESPACE diff --git a/util/hash.h b/util/hash.h index 6e71ad438..cf1fda3f6 100644 --- a/util/hash.h +++ b/util/hash.h @@ -63,6 +63,10 @@ inline uint64_t NPHash64(const char* data, size_t n) { #endif } +// Convenient and equivalent version of Hash128 without depending on 128-bit +// scalars +void Hash2x64(const char* data, size_t n, uint64_t* high64, uint64_t* low64); + // Stable/persistent 32-bit hash. Moderate quality and high speed on // small inputs. // TODO: consider rename to Hash32 diff --git a/util/hash_test.cc b/util/hash_test.cc index abb4ea719..231d06fce 100644 --- a/util/hash_test.cc +++ b/util/hash_test.cc @@ -21,6 +21,7 @@ using ROCKSDB_NAMESPACE::EncodeFixed32; using ROCKSDB_NAMESPACE::GetSliceHash64; using ROCKSDB_NAMESPACE::Hash; using ROCKSDB_NAMESPACE::Hash128; +using ROCKSDB_NAMESPACE::Hash2x64; using ROCKSDB_NAMESPACE::Hash64; using ROCKSDB_NAMESPACE::Lower32of64; using ROCKSDB_NAMESPACE::Lower64of128; @@ -286,6 +287,12 @@ TEST(HashTest, Hash128Misc) { // Must be same as unseeded Hash128 and GetSliceHash128 EXPECT_EQ(here, Hash128(str.data(), size)); EXPECT_EQ(here, GetSliceHash128(Slice(str.data(), size))); + { + uint64_t hi, lo; + Hash2x64(str.data(), size, &hi, &lo); + EXPECT_EQ(Lower64of128(here), lo); + EXPECT_EQ(Upper64of128(here), hi); + } // Upper and Lower must reconstruct hash EXPECT_EQ(here, diff --git a/util/string_util.h b/util/string_util.h index 195ae8b0b..1263deb47 100644 --- a/util/string_util.h +++ b/util/string_util.h @@ -42,6 +42,19 @@ extern void AppendEscapedStringTo(std::string* str, const Slice& value); // Return a string printout of "num" extern std::string NumberToString(uint64_t num); +// Put n digits from v in base kBase to (*buf)[0] to (*buf)[n-1] and +// advance *buf to the position after what was written. +template +inline void PutBaseChars(char** buf, size_t n, uint64_t v, bool uppercase) { + const char* digitChars = uppercase ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + : "0123456789abcdefghijklmnopqrstuvwxyz"; + for (size_t i = n; i > 0; --i) { + (*buf)[i - 1] = digitChars[v % kBase]; + v /= kBase; + } + *buf += n; +} + // Return a human-readable version of num. // for num >= 10.000, prints "xxK" // for num >= 10.000.000, prints "xxM" diff --git a/utilities/env_librados.cc b/utilities/env_librados.cc index 5842edbc7..53d05f856 100644 --- a/utilities/env_librados.cc +++ b/utilities/env_librados.cc @@ -861,25 +861,6 @@ librados::IoCtx* EnvLibrados::_GetIoctx(const std::string& fpath) { /************************************************************ public functions ************************************************************/ -/** - * @brief generate unique id - * @details Combine system time and random number. - * @return [description] - */ -std::string EnvLibrados::GenerateUniqueId() { - Random64 r(time(nullptr)); - uint64_t random_uuid_portion = - r.Uniform(std::numeric_limits::max()); - uint64_t nanos_uuid_portion = NowNanos(); - char uuid2[200]; - snprintf(uuid2, - 200, - "%16lx-%16lx", - (unsigned long)nanos_uuid_portion, - (unsigned long)random_uuid_portion); - return uuid2; -} - /** * @brief create a new sequential read file handler * @details it will check the existence of fname