diff --git a/CMakeLists.txt b/CMakeLists.txt index 8622242aa..f81e0ca4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,6 +60,13 @@ option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, if (WITH_WINDOWS_UTF8_FILENAMES) add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES) endif() +# third-party/folly is only validated to work on Linux and Windows for now. +# So only turn it on there by default. +if(CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_SYSTEM_NAME MATCHES "Windows") + option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" ON) +else() + option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF) +endif() if(MSVC) # Defaults currently different for GFLAGS. # We will address find_package work a little later @@ -462,6 +469,9 @@ endif() include_directories(${PROJECT_SOURCE_DIR}) include_directories(${PROJECT_SOURCE_DIR}/include) include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src) +if(WITH_FOLLY_DISTRIBUTED_MUTEX) + include_directories(${PROJECT_SOURCE_DIR}/third-party/folly) +endif() find_package(Threads REQUIRED) # Main library source code @@ -738,6 +748,15 @@ else() env/io_posix.cc) endif() +if(WITH_FOLLY_DISTRIBUTED_MUTEX) + list(APPEND SOURCES + third-party/folly/folly/detail/Futex.cpp + third-party/folly/folly/synchronization/AtomicNotification.cpp + third-party/folly/folly/synchronization/DistributedMutex.cpp + third-party/folly/folly/synchronization/ParkingLot.cpp + third-party/folly/folly/synchronization/WaitOptions.cpp) +endif() + set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX}) set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX}) set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB}) @@ -1009,6 +1028,10 @@ if(WITH_TESTS) list(APPEND TESTS utilities/env_librados_test.cc) endif() + if(WITH_FOLLY_DISTRIBUTED_MUTEX) + list(APPEND TESTS third-party/folly/folly/synchronization/test/DistributedMutexTest.cpp) + endif() + set(BENCHMARKS cache/cache_bench.cc memtable/memtablerep_bench.cc diff --git a/Makefile b/Makefile index 1718309cb..ccca3ac5e 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,7 @@ endif ifeq ($(MAKECMDGOALS),rocksdbjavastaticreleasedocker) ifneq ($(DEBUG_LEVEL),2) - DEBUG_LEVEL=0 + DEBUG_LEVEL=0 endif endif @@ -304,6 +304,10 @@ ifndef DISABLE_JEMALLOC PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE) endif +ifndef USE_FOLLY_DISTRIBUTED_MUTEX + USE_FOLLY_DISTRIBUTED_MUTEX=0 +endif + export GTEST_THROW_ON_FAILURE=1 export GTEST_HAS_EXCEPTIONS=1 GTEST_DIR = ./third-party/gtest-1.7.0/fused-src @@ -316,6 +320,18 @@ else PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR) endif +ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1) + FOLLY_DIR = ./third-party/folly + # AIX: pre-defined system headers are surrounded by an extern "C" block + ifeq ($(PLATFORM), OS_AIX) + PLATFORM_CCFLAGS += -I$(FOLLY_DIR) + PLATFORM_CXXFLAGS += -I$(FOLLY_DIR) + else + PLATFORM_CCFLAGS += -isystem $(FOLLY_DIR) + PLATFORM_CXXFLAGS += -isystem $(FOLLY_DIR) + endif +endif + # This (the first rule) must depend on "all". default: all @@ -402,6 +418,9 @@ endif LIBOBJECTS += $(TOOL_LIB_SOURCES:.cc=.o) MOCKOBJECTS = $(MOCK_LIB_SOURCES:.cc=.o) +ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1) + FOLLYOBJECTS = $(FOLLY_SOURCES:.cpp=.o) +endif GTEST = $(GTEST_DIR)/gtest/gtest-all.o TESTUTIL = ./test_util/testutil.o @@ -569,6 +588,10 @@ TESTS = \ block_cache_tracer_test \ block_cache_trace_analyzer_test \ +ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1) + TESTS += folly_synchronization_distributed_mutex_test +endif + PARALLEL_TEST = \ backupable_db_test \ db_bloom_filter_test \ @@ -1120,6 +1143,11 @@ trace_analyzer: tools/trace_analyzer.o $(ANALYZETOOLOBJECTS) $(LIBOBJECTS) block_cache_trace_analyzer: tools/block_cache_analyzer/block_cache_trace_analyzer_tool.o $(ANALYZETOOLOBJECTS) $(LIBOBJECTS) $(AM_LINK) +ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1) +folly_synchronization_distributed_mutex_test: $(LIBOBJECTS) $(TESTHARNESS) $(FOLLYOBJECTS) third-party/folly/folly/synchronization/test/DistributedMutexTest.o + $(AM_LINK) +endif + cache_bench: cache/cache_bench.o $(LIBOBJECTS) $(TESTUTIL) $(AM_LINK) diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 4a52c6cdd..7b18a5d5f 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -150,6 +150,9 @@ case "$TARGET_OS" in PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic" fi PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" + if test -z "$USE_FOLLY_DISTRIBUTED_MUTEX"; then + USE_FOLLY_DISTRIBUTED_MUTEX=1 + fi # PORT_FILES=port/linux/linux_specific.cc ;; SunOS) @@ -661,3 +664,6 @@ if test -n "$WITH_JEMALLOC_FLAG"; then echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT" fi echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT" +if test -n "$USE_FOLLY_DISTRIBUTED_MUTEX"; then + echo "USE_FOLLY_DISTRIBUTED_MUTEX=$USE_FOLLY_DISTRIBUTED_MUTEX" >> "$OUTPUT" +fi diff --git a/build_tools/fbcode_config.sh b/build_tools/fbcode_config.sh index 4415f87da..c2c39db48 100644 --- a/build_tools/fbcode_config.sh +++ b/build_tools/fbcode_config.sh @@ -159,4 +159,6 @@ else LUA_LIB=" $LUA_PATH/lib/liblua_pic.a" fi +USE_FOLLY_DISTRIBUTED_MUTEX=1 + export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB diff --git a/build_tools/fbcode_config_platform007.sh b/build_tools/fbcode_config_platform007.sh index 1a1e42081..9da23fd84 100644 --- a/build_tools/fbcode_config_platform007.sh +++ b/build_tools/fbcode_config_platform007.sh @@ -155,4 +155,6 @@ VALGRIND_VER="$VALGRIND_BASE/bin/" LUA_PATH= LUA_LIB= +USE_FOLLY_DISTRIBUTED_MUTEX=1 + export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB diff --git a/src.mk b/src.mk index 6d1d655c7..8ebc0bee9 100644 --- a/src.mk +++ b/src.mk @@ -263,6 +263,13 @@ TEST_LIB_SOURCES = \ test_util/testutil.cc \ utilities/cassandra/test_utils.cc \ +FOLLY_SOURCES = \ + third-party/folly/folly/detail/Futex.cpp \ + third-party/folly/folly/synchronization/AtomicNotification.cpp \ + third-party/folly/folly/synchronization/DistributedMutex.cpp \ + third-party/folly/folly/synchronization/ParkingLot.cpp \ + third-party/folly/folly/synchronization/WaitOptions.cpp \ + MAIN_SOURCES = \ cache/cache_bench.cc \ cache/cache_test.cc \ diff --git a/third-party/folly/folly/CPortability.h b/third-party/folly/folly/CPortability.h new file mode 100644 index 000000000..3ce3a7785 --- /dev/null +++ b/third-party/folly/folly/CPortability.h @@ -0,0 +1,15 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +/** + * Macro for marking functions as having public visibility. + */ +#if defined(__GNUC__) +#define FOLLY_EXPORT __attribute__((__visibility__("default"))) +#else +#define FOLLY_EXPORT +#endif diff --git a/third-party/folly/folly/ConstexprMath.h b/third-party/folly/folly/ConstexprMath.h new file mode 100644 index 000000000..b125c5f42 --- /dev/null +++ b/third-party/folly/folly/ConstexprMath.h @@ -0,0 +1,17 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +namespace folly { +template +constexpr T constexpr_max(T a) { + return a; +} +template +constexpr T constexpr_max(T a, T b, Ts... ts) { + return b < a ? constexpr_max(a, ts...) : constexpr_max(b, ts...); +} +} // namespace folly diff --git a/third-party/folly/folly/Indestructible.h b/third-party/folly/folly/Indestructible.h new file mode 100644 index 000000000..68249d865 --- /dev/null +++ b/third-party/folly/folly/Indestructible.h @@ -0,0 +1,166 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include + +namespace folly { + +/*** + * Indestructible + * + * When you need a Meyers singleton that will not get destructed, even at + * shutdown, and you also want the object stored inline. + * + * Use like: + * + * void doSomethingWithExpensiveData(); + * + * void doSomethingWithExpensiveData() { + * static const Indestructible> data{ + * map{{"key1", 17}, {"key2", 19}, {"key3", 23}}, + * }; + * callSomethingTakingAMapByRef(*data); + * } + * + * This should be used only for Meyers singletons, and, even then, only when + * the instance does not need to be destructed ever. + * + * This should not be used more generally, e.g., as member fields, etc. + * + * This is designed as an alternative, but with one fewer allocation at + * construction time and one fewer pointer dereference at access time, to the + * Meyers singleton pattern of: + * + * void doSomethingWithExpensiveData() { + * static const auto data = // never `delete`d + * new map{{"key1", 17}, {"key2", 19}, {"key3", 23}}; + * callSomethingTakingAMapByRef(*data); + * } + */ + +template +class Indestructible final { + public: + template + constexpr Indestructible() noexcept(noexcept(T())) {} + + /** + * Constructor accepting a single argument by forwarding reference, this + * allows using list initialzation without the overhead of things like + * in_place, etc and also works with std::initializer_list constructors + * which can't be deduced, the default parameter helps there. + * + * auto i = folly::Indestructible>{{{1, 2}}}; + * + * This provides convenience + * + * There are two versions of this constructor - one for when the element is + * implicitly constructible from the given argument and one for when the + * type is explicitly but not implicitly constructible from the given + * argument. + */ + template < + typename U = T, + _t::value>>* = nullptr, + _t, remove_cvref_t>::value>>* = + nullptr, + _t::value>>* = nullptr> + explicit constexpr Indestructible(U&& u) noexcept( + noexcept(T(std::declval()))) + : storage_(std::forward(u)) {} + template < + typename U = T, + _t::value>>* = nullptr, + _t, remove_cvref_t>::value>>* = + nullptr, + _t::value>>* = nullptr> + /* implicit */ constexpr Indestructible(U&& u) noexcept( + noexcept(T(std::declval()))) + : storage_(std::forward(u)) {} + + template ()...))> + explicit constexpr Indestructible(Args&&... args) noexcept( + noexcept(T(std::declval()...))) + : storage_(std::forward(args)...) {} + template < + typename U, + typename... Args, + typename = decltype( + T(std::declval&>(), + std::declval()...))> + explicit constexpr Indestructible(std::initializer_list il, Args... args) noexcept( + noexcept( + T(std::declval&>(), + std::declval()...))) + : storage_(il, std::forward(args)...) {} + + ~Indestructible() = default; + + Indestructible(Indestructible const&) = delete; + Indestructible& operator=(Indestructible const&) = delete; + + Indestructible(Indestructible&& other) noexcept( + noexcept(T(std::declval()))) + : storage_(std::move(other.storage_.value)) { + other.erased_ = true; + } + Indestructible& operator=(Indestructible&& other) noexcept( + noexcept(T(std::declval()))) { + storage_.value = std::move(other.storage_.value); + other.erased_ = true; + } + + T* get() noexcept { + check(); + return &storage_.value; + } + T const* get() const noexcept { + check(); + return &storage_.value; + } + T& operator*() noexcept { + return *get(); + } + T const& operator*() const noexcept { + return *get(); + } + T* operator->() noexcept { + return get(); + } + T const* operator->() const noexcept { + return get(); + } + + private: + void check() const noexcept { + assert(!erased_); + } + + union Storage { + T value; + + template + constexpr Storage() noexcept(noexcept(T())) : value() {} + + template ()...))> + explicit constexpr Storage(Args&&... args) noexcept( + noexcept(T(std::declval()...))) + : value(std::forward(args)...) {} + + ~Storage() {} + }; + + Storage storage_{}; + bool erased_{false}; +}; +} // namespace folly diff --git a/third-party/folly/folly/Optional.h b/third-party/folly/folly/Optional.h new file mode 100644 index 000000000..ee12467dd --- /dev/null +++ b/third-party/folly/folly/Optional.h @@ -0,0 +1,570 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +/* + * Optional - For conditional initialization of values, like boost::optional, + * but with support for move semantics and emplacement. Reference type support + * has not been included due to limited use cases and potential confusion with + * semantics of assignment: Assigning to an optional reference could quite + * reasonably copy its value or redirect the reference. + * + * Optional can be useful when a variable might or might not be needed: + * + * Optional maybeLogger = ...; + * if (maybeLogger) { + * maybeLogger->log("hello"); + * } + * + * Optional enables a 'null' value for types which do not otherwise have + * nullability, especially useful for parameter passing: + * + * void testIterator(const unique_ptr& it, + * initializer_list idsExpected, + * Optional> ranksExpected = none) { + * for (int i = 0; it->next(); ++i) { + * EXPECT_EQ(it->doc().id(), idsExpected[i]); + * if (ranksExpected) { + * EXPECT_EQ(it->doc().rank(), (*ranksExpected)[i]); + * } + * } + * } + * + * Optional models OptionalPointee, so calling 'get_pointer(opt)' will return a + * pointer to nullptr if the 'opt' is empty, and a pointer to the value if it is + * not: + * + * Optional maybeInt = ...; + * if (int* v = get_pointer(maybeInt)) { + * cout << *v << endl; + * } + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace folly { + +template +class Optional; + +namespace detail { +template +struct OptionalPromiseReturn; +} // namespace detail + +struct None { + enum class _secret { _token }; + + /** + * No default constructor to support both `op = {}` and `op = none` + * as syntax for clearing an Optional, just like std::nullopt_t. + */ + constexpr explicit None(_secret) {} +}; +constexpr None none{None::_secret::_token}; + +class FOLLY_EXPORT OptionalEmptyException : public std::runtime_error { + public: + OptionalEmptyException() + : std::runtime_error("Empty Optional cannot be unwrapped") {} +}; + +template +class Optional { + public: + typedef Value value_type; + + static_assert( + !std::is_reference::value, + "Optional may not be used with reference types"); + static_assert( + !std::is_abstract::value, + "Optional may not be used with abstract types"); + + Optional() noexcept {} + + Optional(const Optional& src) noexcept( + std::is_nothrow_copy_constructible::value) { + if (src.hasValue()) { + construct(src.value()); + } + } + + Optional(Optional&& src) noexcept( + std::is_nothrow_move_constructible::value) { + if (src.hasValue()) { + construct(std::move(src.value())); + src.clear(); + } + } + + /* implicit */ Optional(const None&) noexcept {} + + /* implicit */ Optional(Value&& newValue) noexcept( + std::is_nothrow_move_constructible::value) { + construct(std::move(newValue)); + } + + /* implicit */ Optional(const Value& newValue) noexcept( + std::is_nothrow_copy_constructible::value) { + construct(newValue); + } + + template + explicit Optional(in_place_t, Args&&... args) noexcept( + std::is_nothrow_constructible::value) + : Optional{PrivateConstructor{}, std::forward(args)...} {} + + template + explicit Optional( + in_place_t, + std::initializer_list il, + Args&&... args) noexcept(std:: + is_nothrow_constructible< + Value, + std::initializer_list, + Args...>::value) + : Optional{PrivateConstructor{}, il, std::forward(args)...} {} + + // Used only when an Optional is used with coroutines on MSVC + /* implicit */ Optional(const detail::OptionalPromiseReturn& p) + : Optional{} { + p.promise_->value_ = this; + } + + void assign(const None&) { + clear(); + } + + void assign(Optional&& src) { + if (this != &src) { + if (src.hasValue()) { + assign(std::move(src.value())); + src.clear(); + } else { + clear(); + } + } + } + + void assign(const Optional& src) { + if (src.hasValue()) { + assign(src.value()); + } else { + clear(); + } + } + + void assign(Value&& newValue) { + if (hasValue()) { + storage_.value = std::move(newValue); + } else { + construct(std::move(newValue)); + } + } + + void assign(const Value& newValue) { + if (hasValue()) { + storage_.value = newValue; + } else { + construct(newValue); + } + } + + Optional& operator=(None) noexcept { + reset(); + return *this; + } + + template + Optional& operator=(Arg&& arg) { + assign(std::forward(arg)); + return *this; + } + + Optional& operator=(Optional&& other) noexcept( + std::is_nothrow_move_assignable::value) { + assign(std::move(other)); + return *this; + } + + Optional& operator=(const Optional& other) noexcept( + std::is_nothrow_copy_assignable::value) { + assign(other); + return *this; + } + + template + Value& emplace(Args&&... args) { + clear(); + construct(std::forward(args)...); + return value(); + } + + template + typename std::enable_if< + std::is_constructible&, Args&&...>::value, + Value&>::type + emplace(std::initializer_list ilist, Args&&... args) { + clear(); + construct(ilist, std::forward(args)...); + return value(); + } + + void reset() noexcept { + storage_.clear(); + } + + void clear() noexcept { + reset(); + } + + void swap(Optional& that) noexcept(IsNothrowSwappable::value) { + if (hasValue() && that.hasValue()) { + using std::swap; + swap(value(), that.value()); + } else if (hasValue()) { + that.emplace(std::move(value())); + reset(); + } else if (that.hasValue()) { + emplace(std::move(that.value())); + that.reset(); + } + } + + const Value& value() const& { + require_value(); + return storage_.value; + } + + Value& value() & { + require_value(); + return storage_.value; + } + + Value&& value() && { + require_value(); + return std::move(storage_.value); + } + + const Value&& value() const&& { + require_value(); + return std::move(storage_.value); + } + + const Value* get_pointer() const& { + return storage_.hasValue ? &storage_.value : nullptr; + } + Value* get_pointer() & { + return storage_.hasValue ? &storage_.value : nullptr; + } + Value* get_pointer() && = delete; + + bool has_value() const noexcept { + return storage_.hasValue; + } + + bool hasValue() const noexcept { + return has_value(); + } + + explicit operator bool() const noexcept { + return has_value(); + } + + const Value& operator*() const& { + return value(); + } + Value& operator*() & { + return value(); + } + const Value&& operator*() const&& { + return std::move(value()); + } + Value&& operator*() && { + return std::move(value()); + } + + const Value* operator->() const { + return &value(); + } + Value* operator->() { + return &value(); + } + + // Return a copy of the value if set, or a given default if not. + template + Value value_or(U&& dflt) const& { + if (storage_.hasValue) { + return storage_.value; + } + + return std::forward(dflt); + } + + template + Value value_or(U&& dflt) && { + if (storage_.hasValue) { + return std::move(storage_.value); + } + + return std::forward(dflt); + } + + private: + template + friend Optional<_t>> make_optional(T&&); + template + friend Optional make_optional(Args&&... args); + template + friend Optional make_optional(std::initializer_list, As&&...); + + /** + * Construct the optional in place, this is duplicated as a non-explicit + * constructor to allow returning values that are non-movable from + * make_optional using list initialization. + * + * Until C++17, at which point this will become unnecessary because of + * specified prvalue elision. + */ + struct PrivateConstructor { + explicit PrivateConstructor() = default; + }; + template + Optional(PrivateConstructor, Args&&... args) noexcept( + std::is_constructible::value) { + construct(std::forward(args)...); + } + + void require_value() const { + if (!storage_.hasValue) { + throw OptionalEmptyException{}; + } + } + + template + void construct(Args&&... args) { + const void* ptr = &storage_.value; + // For supporting const types. + new (const_cast(ptr)) Value(std::forward(args)...); + storage_.hasValue = true; + } + + struct StorageTriviallyDestructible { + union { + char emptyState; + Value value; + }; + bool hasValue; + + StorageTriviallyDestructible() + : emptyState('\0'), hasValue{false} {} + void clear() { + hasValue = false; + } + }; + + struct StorageNonTriviallyDestructible { + union { + char emptyState; + Value value; + }; + bool hasValue; + + StorageNonTriviallyDestructible() : hasValue{false} {} + ~StorageNonTriviallyDestructible() { + clear(); + } + + void clear() { + if (hasValue) { + hasValue = false; + value.~Value(); + } + } + }; + + using Storage = typename std::conditional< + std::is_trivially_destructible::value, + StorageTriviallyDestructible, + StorageNonTriviallyDestructible>::type; + + Storage storage_; +}; + +template +const T* get_pointer(const Optional& opt) { + return opt.get_pointer(); +} + +template +T* get_pointer(Optional& opt) { + return opt.get_pointer(); +} + +template +void swap(Optional& a, Optional& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); +} + +template +Optional<_t>> make_optional(T&& v) { + using PrivateConstructor = + typename folly::Optional<_t>>::PrivateConstructor; + return {PrivateConstructor{}, std::forward(v)}; +} + +template +folly::Optional make_optional(Args&&... args) { + using PrivateConstructor = typename folly::Optional::PrivateConstructor; + return {PrivateConstructor{}, std::forward(args)...}; +} + +template +folly::Optional make_optional( + std::initializer_list il, + Args&&... args) { + using PrivateConstructor = typename folly::Optional::PrivateConstructor; + return {PrivateConstructor{}, il, std::forward(args)...}; +} + +/////////////////////////////////////////////////////////////////////////////// +// Comparisons. + +template +bool operator==(const Optional& a, const V& b) { + return a.hasValue() && a.value() == b; +} + +template +bool operator!=(const Optional& a, const V& b) { + return !(a == b); +} + +template +bool operator==(const U& a, const Optional& b) { + return b.hasValue() && b.value() == a; +} + +template +bool operator!=(const U& a, const Optional& b) { + return !(a == b); +} + +template +bool operator==(const Optional& a, const Optional& b) { + if (a.hasValue() != b.hasValue()) { + return false; + } + if (a.hasValue()) { + return a.value() == b.value(); + } + return true; +} + +template +bool operator!=(const Optional& a, const Optional& b) { + return !(a == b); +} + +template +bool operator<(const Optional& a, const Optional& b) { + if (a.hasValue() != b.hasValue()) { + return a.hasValue() < b.hasValue(); + } + if (a.hasValue()) { + return a.value() < b.value(); + } + return false; +} + +template +bool operator>(const Optional& a, const Optional& b) { + return b < a; +} + +template +bool operator<=(const Optional& a, const Optional& b) { + return !(b < a); +} + +template +bool operator>=(const Optional& a, const Optional& b) { + return !(a < b); +} + +// Suppress comparability of Optional with T, despite implicit conversion. +template +bool operator<(const Optional&, const V& other) = delete; +template +bool operator<=(const Optional&, const V& other) = delete; +template +bool operator>=(const Optional&, const V& other) = delete; +template +bool operator>(const Optional&, const V& other) = delete; +template +bool operator<(const V& other, const Optional&) = delete; +template +bool operator<=(const V& other, const Optional&) = delete; +template +bool operator>=(const V& other, const Optional&) = delete; +template +bool operator>(const V& other, const Optional&) = delete; + +// Comparisons with none +template +bool operator==(const Optional& a, None) noexcept { + return !a.hasValue(); +} +template +bool operator==(None, const Optional& a) noexcept { + return !a.hasValue(); +} +template +bool operator<(const Optional&, None) noexcept { + return false; +} +template +bool operator<(None, const Optional& a) noexcept { + return a.hasValue(); +} +template +bool operator>(const Optional& a, None) noexcept { + return a.hasValue(); +} +template +bool operator>(None, const Optional&) noexcept { + return false; +} +template +bool operator<=(None, const Optional&) noexcept { + return true; +} +template +bool operator<=(const Optional& a, None) noexcept { + return !a.hasValue(); +} +template +bool operator>=(const Optional&, None) noexcept { + return true; +} +template +bool operator>=(None, const Optional& a) noexcept { + return !a.hasValue(); +} + +/////////////////////////////////////////////////////////////////////////////// + +} // namespace folly diff --git a/third-party/folly/folly/Portability.h b/third-party/folly/folly/Portability.h new file mode 100644 index 000000000..2c6544c19 --- /dev/null +++ b/third-party/folly/folly/Portability.h @@ -0,0 +1,74 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#if defined(__arm__) +#define FOLLY_ARM 1 +#else +#define FOLLY_ARM 0 +#endif + +#if defined(__x86_64__) || defined(_M_X64) +#define FOLLY_X64 1 +#else +#define FOLLY_X64 0 +#endif + +#if defined(__aarch64__) +#define FOLLY_AARCH64 1 +#else +#define FOLLY_AARCH64 0 +#endif + +#if defined(__powerpc64__) +#define FOLLY_PPC64 1 +#else +#define FOLLY_PPC64 0 +#endif + +#if defined(__has_builtin) +#define FOLLY_HAS_BUILTIN(...) __has_builtin(__VA_ARGS__) +#else +#define FOLLY_HAS_BUILTIN(...) 0 +#endif + +#if defined(__has_cpp_attribute) +#if __has_cpp_attribute(nodiscard) +#define FOLLY_NODISCARD [[nodiscard]] +#endif +#endif +#if !defined FOLLY_NODISCARD +#if defined(_MSC_VER) && (_MSC_VER >= 1700) +#define FOLLY_NODISCARD _Check_return_ +#elif defined(__GNUC__) +#define FOLLY_NODISCARD __attribute__((__warn_unused_result__)) +#else +#define FOLLY_NODISCARD +#endif +#endif + +namespace folly { +constexpr bool kIsArchArm = FOLLY_ARM == 1; +constexpr bool kIsArchAmd64 = FOLLY_X64 == 1; +constexpr bool kIsArchAArch64 = FOLLY_AARCH64 == 1; +constexpr bool kIsArchPPC64 = FOLLY_PPC64 == 1; +} // namespace folly + +namespace folly { +#ifdef NDEBUG +constexpr auto kIsDebug = false; +#else +constexpr auto kIsDebug = true; +#endif +} // namespace folly + +namespace folly { +#if defined(_MSC_VER) +constexpr bool kIsMsvc = true; +#else +constexpr bool kIsMsvc = false; +#endif +} // namespace folly diff --git a/third-party/folly/folly/ScopeGuard.h b/third-party/folly/folly/ScopeGuard.h new file mode 100644 index 000000000..711344063 --- /dev/null +++ b/third-party/folly/folly/ScopeGuard.h @@ -0,0 +1,54 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include +#include + +namespace folly { +namespace scope_guard_detail { +template +class ScopeGuardImpl { + public: + explicit ScopeGuardImpl(F&& f) : f_{std::forward(f)} {} + ~ScopeGuardImpl() { + f_(); + } + + private: + F f_; +}; + +enum class ScopeGuardEnum {}; +template >> +ScopeGuardImpl operator+(ScopeGuardEnum, Func&& func) { + return ScopeGuardImpl{std::forward(func)}; +} +} // namespace scope_guard_detail +} // namespace folly + +/** + * FB_ANONYMOUS_VARIABLE(str) introduces an identifier starting with + * str and ending with a number that varies with the line. + */ +#ifndef FB_ANONYMOUS_VARIABLE +#define FB_CONCATENATE_IMPL(s1, s2) s1##s2 +#define FB_CONCATENATE(s1, s2) FB_CONCATENATE_IMPL(s1, s2) +#ifdef __COUNTER__ +#define FB_ANONYMOUS_VARIABLE(str) \ + FB_CONCATENATE(FB_CONCATENATE(FB_CONCATENATE(str, __COUNTER__), _), __LINE__) +#else +#define FB_ANONYMOUS_VARIABLE(str) FB_CONCATENATE(str, __LINE__) +#endif +#endif + +#ifndef SCOPE_EXIT +#define SCOPE_EXIT \ + auto FB_ANONYMOUS_VARIABLE(SCOPE_EXIT_STATE) = \ + ::folly::scope_guard_detail::ScopeGuardEnum{} + [&]() noexcept +#endif diff --git a/third-party/folly/folly/Traits.h b/third-party/folly/folly/Traits.h new file mode 100644 index 000000000..ea7e1eb1c --- /dev/null +++ b/third-party/folly/folly/Traits.h @@ -0,0 +1,152 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +namespace folly { + +#if !defined(_MSC_VER) +template +struct is_trivially_copyable + : std::integral_constant {}; +#else +template +using is_trivially_copyable = std::is_trivially_copyable; +#endif + +/*** + * _t + * + * Instead of: + * + * using decayed = typename std::decay::type; + * + * With the C++14 standard trait aliases, we could use: + * + * using decayed = std::decay_t; + * + * Without them, we could use: + * + * using decayed = _t>; + * + * Also useful for any other library with template types having dependent + * member types named `type`, like the standard trait types. + */ +template +using _t = typename T::type; + +/** + * type_t + * + * A type alias for the first template type argument. `type_t` is useful for + * controlling class-template and function-template partial specialization. + * + * Example: + * + * template + * class Container { + * public: + * template + * Container( + * type_t()...))>, + * Args&&...); + * }; + * + * void_t + * + * A type alias for `void`. `void_t` is useful for controling class-template + * and function-template partial specialization. + * + * Example: + * + * // has_value_type::value is true if T has a nested type `value_type` + * template + * struct has_value_type + * : std::false_type {}; + * + * template + * struct has_value_type> + * : std::true_type {}; + */ + +/** + * There is a bug in libstdc++, libc++, and MSVC's STL that causes it to + * ignore unused template parameter arguments in template aliases and does not + * cause substitution failures. This defect has been recorded here: + * http://open-std.org/JTC1/SC22/WG21/docs/cwg_defects.html#1558. + * + * This causes the implementation of std::void_t to be buggy, as it is likely + * defined as something like the following: + * + * template + * using void_t = void; + * + * This causes the compiler to ignore all the template arguments and does not + * help when one wants to cause substitution failures. Rather declarations + * which have void_t in orthogonal specializations are treated as the same. + * For example, assuming the possible `T` types are only allowed to have + * either the alias `one` or `two` and never both or none: + * + * template ::one>* = nullptr> + * void foo(T&&) {} + * template ::two>* = nullptr> + * void foo(T&&) {} + * + * The second foo() will be a redefinition because it conflicts with the first + * one; void_t does not cause substitution failures - the template types are + * just ignored. + */ + +namespace traits_detail { +template +struct type_t_ { + using type = T; +}; +} // namespace traits_detail + +template +using type_t = typename traits_detail::type_t_::type; +template +using void_t = type_t; + +/** + * A type trait to remove all const volatile and reference qualifiers on a + * type T + */ +template +struct remove_cvref { + using type = + typename std::remove_cv::type>::type; +}; +template +using remove_cvref_t = typename remove_cvref::type; + +template +struct IsNothrowSwappable + : std::integral_constant< + bool, + std::is_nothrow_move_constructible::value&& noexcept( + std::swap(std::declval(), std::declval()))> {}; + +template +struct Conjunction : std::true_type {}; +template +struct Conjunction : T {}; +template +struct Conjunction + : std::conditional, T>::type {}; + +template +struct Negation : std::integral_constant {}; + +template +using index_constant = std::integral_constant; + +} // namespace folly diff --git a/third-party/folly/folly/Unit.h b/third-party/folly/folly/Unit.h new file mode 100644 index 000000000..c8cb77e2c --- /dev/null +++ b/third-party/folly/folly/Unit.h @@ -0,0 +1,59 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +namespace folly { + +/// In functional programming, the degenerate case is often called "unit". In +/// C++, "void" is often the best analogue. However, because of the syntactic +/// special-casing required for void, it is frequently a liability for template +/// metaprogramming. So, instead of writing specializations to handle cases like +/// SomeContainer, a library author may instead rule that out and simply +/// have library users use SomeContainer. Contained values may be ignored. +/// Much easier. +/// +/// "void" is the type that admits of no values at all. It is not possible to +/// construct a value of this type. +/// "unit" is the type that admits of precisely one unique value. It is +/// possible to construct a value of this type, but it is always the same value +/// every time, so it is uninteresting. +struct Unit { + constexpr bool operator==(const Unit& /*other*/) const { + return true; + } + constexpr bool operator!=(const Unit& /*other*/) const { + return false; + } +}; + +constexpr Unit unit{}; + +template +struct lift_unit { + using type = T; +}; +template <> +struct lift_unit { + using type = Unit; +}; +template +using lift_unit_t = typename lift_unit::type; + +template +struct drop_unit { + using type = T; +}; +template <> +struct drop_unit { + using type = void; +}; +template +using drop_unit_t = typename drop_unit::type; + +} // namespace folly + diff --git a/third-party/folly/folly/Utility.h b/third-party/folly/folly/Utility.h new file mode 100644 index 000000000..7e43bdc2f --- /dev/null +++ b/third-party/folly/folly/Utility.h @@ -0,0 +1,141 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +namespace folly { + +/** + * Backports from C++17 of: + * std::in_place_t + * std::in_place_type_t + * std::in_place_index_t + * std::in_place + * std::in_place_type + * std::in_place_index + */ + +struct in_place_tag {}; +template +struct in_place_type_tag {}; +template +struct in_place_index_tag {}; + +using in_place_t = in_place_tag (&)(in_place_tag); +template +using in_place_type_t = in_place_type_tag (&)(in_place_type_tag); +template +using in_place_index_t = in_place_index_tag (&)(in_place_index_tag); + +inline in_place_tag in_place(in_place_tag = {}) { + return {}; +} +template +inline in_place_type_tag in_place_type(in_place_type_tag = {}) { + return {}; +} +template +inline in_place_index_tag in_place_index(in_place_index_tag = {}) { + return {}; +} + +template +T exchange(T& obj, U&& new_value) { + T old_value = std::move(obj); + obj = std::forward(new_value); + return old_value; +} + +namespace utility_detail { +template +struct make_seq_cat; +template < + template class S, + typename T, + T... Ta, + T... Tb, + T... Tc> +struct make_seq_cat, S, S> { + using type = + S; +}; + +// Not parameterizing by `template class, typename` because +// clang precisely v4.0 fails to compile that. Note that clang v3.9 and v5.0 +// handle that code correctly. +// +// For this to work, `S0` is required to be `Sequence` and `S1` is required +// to be `Sequence`. + +template +struct make_seq { + template + using apply = typename make_seq_cat< + typename make_seq::template apply, + typename make_seq::template apply, + typename make_seq::template apply>::type; +}; +template <> +struct make_seq<1> { + template + using apply = S1; +}; +template <> +struct make_seq<0> { + template + using apply = S0; +}; +} // namespace utility_detail + +// TODO: Remove after upgrading to C++14 baseline + +template +struct integer_sequence { + using value_type = T; + + static constexpr std::size_t size() noexcept { + return sizeof...(Ints); + } +}; + +template +using index_sequence = integer_sequence; + +template +using make_integer_sequence = typename utility_detail::make_seq< + Size>::template apply, integer_sequence>; + +template +using make_index_sequence = make_integer_sequence; +template +using index_sequence_for = make_index_sequence; + +/** + * A simple helper for getting a constant reference to an object. + * + * Example: + * + * std::vector v{1,2,3}; + * // The following two lines are equivalent: + * auto a = const_cast&>(v).begin(); + * auto b = folly::as_const(v).begin(); + * + * Like C++17's std::as_const. See http://wg21.link/p0007 + */ +template +T const& as_const(T& t) noexcept { + return t; +} + +template +void as_const(T const&&) = delete; + +} // namespace folly diff --git a/third-party/folly/folly/chrono/Hardware.h b/third-party/folly/folly/chrono/Hardware.h new file mode 100644 index 000000000..ec7be82e8 --- /dev/null +++ b/third-party/folly/folly/chrono/Hardware.h @@ -0,0 +1,33 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include + +#include +#include + +#if _MSC_VER +extern "C" std::uint64_t __rdtsc(); +#pragma intrinsic(__rdtsc) +#endif + +namespace folly { + +inline std::uint64_t hardware_timestamp() { +#if _MSC_VER + return __rdtsc(); +#elif __GNUC__ && (__i386__ || FOLLY_X64) + return __builtin_ia32_rdtsc(); +#else + // use steady_clock::now() as an approximation for the timestamp counter on + // non-x86 systems + return std::chrono::steady_clock::now().time_since_epoch().count(); +#endif +} + +} // namespace folly + diff --git a/third-party/folly/folly/container/Array.h b/third-party/folly/folly/container/Array.h new file mode 100644 index 000000000..bb3167b97 --- /dev/null +++ b/third-party/folly/folly/container/Array.h @@ -0,0 +1,74 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include +#include + +namespace folly { + +namespace array_detail { +template +struct is_ref_wrapper : std::false_type {}; +template +struct is_ref_wrapper> : std::true_type {}; + +template +using not_ref_wrapper = + folly::Negation::type>>; + +template +struct return_type_helper { + using type = D; +}; +template +struct return_type_helper { + static_assert( + folly::Conjunction...>::value, + "TList cannot contain reference_wrappers when D is void"); + using type = typename std::common_type::type; +}; + +template +using return_type = std:: + array::type, sizeof...(TList)>; +} // namespace array_detail + +template +constexpr array_detail::return_type make_array(TList&&... t) { + using value_type = + typename array_detail::return_type_helper::type; + return {{static_cast(std::forward(t))...}}; +} + +namespace array_detail { +template +inline constexpr auto make_array_with( + MakeItem const& make, + folly::index_sequence) + -> std::array { + return std::array{{make(Index)...}}; +} +} // namespace array_detail + +// make_array_with +// +// Constructs a std::array<..., Size> with elements m(i) for i in [0, Size). +template +constexpr auto make_array_with(MakeItem const& make) + -> decltype(array_detail::make_array_with( + make, + folly::make_index_sequence{})) { + return array_detail::make_array_with( + make, + folly::make_index_sequence{}); +} + +} // namespace folly diff --git a/third-party/folly/folly/detail/Futex-inl.h b/third-party/folly/folly/detail/Futex-inl.h new file mode 100644 index 000000000..3b2a412bf --- /dev/null +++ b/third-party/folly/folly/detail/Futex-inl.h @@ -0,0 +1,117 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +namespace folly { +namespace detail { + +/** Optimal when TargetClock is the same type as Clock. + * + * Otherwise, both Clock::now() and TargetClock::now() must be invoked. */ +template +typename TargetClock::time_point time_point_conv( + std::chrono::time_point const& time) { + using std::chrono::duration_cast; + using TimePoint = std::chrono::time_point; + using TargetDuration = typename TargetClock::duration; + using TargetTimePoint = typename TargetClock::time_point; + if (time == TimePoint::max()) { + return TargetTimePoint::max(); + } else if (std::is_same::value) { + // in place of time_point_cast, which cannot compile without if-constexpr + auto const delta = time.time_since_epoch(); + return TargetTimePoint(duration_cast(delta)); + } else { + // different clocks with different epochs, so non-optimal case + auto const delta = time - Clock::now(); + return TargetClock::now() + duration_cast(delta); + } +} + +/** + * Available overloads, with definitions elsewhere + * + * These functions are treated as ADL-extension points, the templates above + * call these functions without them having being pre-declared. This works + * because ADL lookup finds the definitions of these functions when you pass + * the relevant arguments + */ +int futexWakeImpl( + const Futex* futex, + int count, + uint32_t wakeMask); +FutexResult futexWaitImpl( + const Futex* futex, + uint32_t expected, + std::chrono::system_clock::time_point const* absSystemTime, + std::chrono::steady_clock::time_point const* absSteadyTime, + uint32_t waitMask); + +int futexWakeImpl( + const Futex* futex, + int count, + uint32_t wakeMask); +FutexResult futexWaitImpl( + const Futex* futex, + uint32_t expected, + std::chrono::system_clock::time_point const* absSystemTime, + std::chrono::steady_clock::time_point const* absSteadyTime, + uint32_t waitMask); + +template +typename std::enable_if::type +futexWaitImpl( + Futex* futex, + uint32_t expected, + Deadline const& deadline, + uint32_t waitMask) { + return futexWaitImpl(futex, expected, nullptr, &deadline, waitMask); +} + +template +typename std::enable_if::type +futexWaitImpl( + Futex* futex, + uint32_t expected, + Deadline const& deadline, + uint32_t waitMask) { + return futexWaitImpl(futex, expected, &deadline, nullptr, waitMask); +} + +template +FutexResult +futexWait(const Futex* futex, uint32_t expected, uint32_t waitMask) { + auto rv = futexWaitImpl(futex, expected, nullptr, nullptr, waitMask); + assert(rv != FutexResult::TIMEDOUT); + return rv; +} + +template +int futexWake(const Futex* futex, int count, uint32_t wakeMask) { + return futexWakeImpl(futex, count, wakeMask); +} + +template +FutexResult futexWaitUntil( + const Futex* futex, + uint32_t expected, + std::chrono::time_point const& deadline, + uint32_t waitMask) { + using Target = typename std::conditional< + Clock::is_steady, + std::chrono::steady_clock, + std::chrono::system_clock>::type; + auto const converted = time_point_conv(deadline); + return converted == Target::time_point::max() + ? futexWaitImpl(futex, expected, nullptr, nullptr, waitMask) + : futexWaitImpl(futex, expected, converted, waitMask); +} + +} // namespace detail +} // namespace folly diff --git a/third-party/folly/folly/detail/Futex.cpp b/third-party/folly/folly/detail/Futex.cpp new file mode 100644 index 000000000..208578a90 --- /dev/null +++ b/third-party/folly/folly/detail/Futex.cpp @@ -0,0 +1,263 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include +#include +#include +#include +#include + +#include + +#ifdef __linux__ +#include +#endif + +#ifndef _WIN32 +#include +#endif + +using namespace std::chrono; + +namespace folly { +namespace detail { + +namespace { + +//////////////////////////////////////////////////// +// native implementation using the futex() syscall + +#ifdef __linux__ + +/// Certain toolchains (like Android's) don't include the full futex API in +/// their headers even though they support it. Make sure we have our constants +/// even if the headers don't have them. +#ifndef FUTEX_WAIT_BITSET +#define FUTEX_WAIT_BITSET 9 +#endif +#ifndef FUTEX_WAKE_BITSET +#define FUTEX_WAKE_BITSET 10 +#endif +#ifndef FUTEX_PRIVATE_FLAG +#define FUTEX_PRIVATE_FLAG 128 +#endif +#ifndef FUTEX_CLOCK_REALTIME +#define FUTEX_CLOCK_REALTIME 256 +#endif + +int nativeFutexWake(const void* addr, int count, uint32_t wakeMask) { + int rv = syscall( + __NR_futex, + addr, /* addr1 */ + FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG, /* op */ + count, /* val */ + nullptr, /* timeout */ + nullptr, /* addr2 */ + wakeMask); /* val3 */ + + /* NOTE: we ignore errors on wake for the case of a futex + guarding its own destruction, similar to this + glibc bug with sem_post/sem_wait: + https://sourceware.org/bugzilla/show_bug.cgi?id=12674 */ + if (rv < 0) { + return 0; + } + return rv; +} + +template +struct timespec timeSpecFromTimePoint(time_point absTime) { + auto epoch = absTime.time_since_epoch(); + if (epoch.count() < 0) { + // kernel timespec_valid requires non-negative seconds and nanos in [0,1G) + epoch = Clock::duration::zero(); + } + + // timespec-safe seconds and nanoseconds; + // chrono::{nano,}seconds are `long long int` + // whereas timespec uses smaller types + using time_t_seconds = duration; + using long_nanos = duration; + + auto secs = duration_cast(epoch); + auto nanos = duration_cast(epoch - secs); + struct timespec result = {secs.count(), nanos.count()}; + return result; +} + +FutexResult nativeFutexWaitImpl( + const void* addr, + uint32_t expected, + system_clock::time_point const* absSystemTime, + steady_clock::time_point const* absSteadyTime, + uint32_t waitMask) { + assert(absSystemTime == nullptr || absSteadyTime == nullptr); + + int op = FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG; + struct timespec ts; + struct timespec* timeout = nullptr; + + if (absSystemTime != nullptr) { + op |= FUTEX_CLOCK_REALTIME; + ts = timeSpecFromTimePoint(*absSystemTime); + timeout = &ts; + } else if (absSteadyTime != nullptr) { + ts = timeSpecFromTimePoint(*absSteadyTime); + timeout = &ts; + } + + // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET requires an absolute timeout + // value - http://locklessinc.com/articles/futex_cheat_sheet/ + int rv = syscall( + __NR_futex, + addr, /* addr1 */ + op, /* op */ + expected, /* val */ + timeout, /* timeout */ + nullptr, /* addr2 */ + waitMask); /* val3 */ + + if (rv == 0) { + return FutexResult::AWOKEN; + } else { + switch (errno) { + case ETIMEDOUT: + assert(timeout != nullptr); + return FutexResult::TIMEDOUT; + case EINTR: + return FutexResult::INTERRUPTED; + case EWOULDBLOCK: + return FutexResult::VALUE_CHANGED; + default: + assert(false); + // EINVAL, EACCESS, or EFAULT. EINVAL means there was an invalid + // op (should be impossible) or an invalid timeout (should have + // been sanitized by timeSpecFromTimePoint). EACCESS or EFAULT + // means *addr points to invalid memory, which is unlikely because + // the caller should have segfaulted already. We can either + // crash, or return a value that lets the process continue for + // a bit. We choose the latter. VALUE_CHANGED probably turns the + // caller into a spin lock. + return FutexResult::VALUE_CHANGED; + } + } +} + +#endif // __linux__ + +/////////////////////////////////////////////////////// +// compatibility implementation using standard C++ API + +using Lot = ParkingLot; +Lot parkingLot; + +int emulatedFutexWake(const void* addr, int count, uint32_t waitMask) { + int woken = 0; + parkingLot.unpark(addr, [&](const uint32_t& mask) { + if ((mask & waitMask) == 0) { + return UnparkControl::RetainContinue; + } + assert(count > 0); + count--; + woken++; + return count > 0 ? UnparkControl::RemoveContinue + : UnparkControl::RemoveBreak; + }); + return woken; +} + +template +FutexResult emulatedFutexWaitImpl( + F* futex, + uint32_t expected, + system_clock::time_point const* absSystemTime, + steady_clock::time_point const* absSteadyTime, + uint32_t waitMask) { + static_assert( + std::is_same>::value || + std::is_same>::value, + "Type F must be either Futex or Futex"); + ParkResult res; + if (absSystemTime) { + res = parkingLot.park_until( + futex, + waitMask, + [&] { return *futex == expected; }, + [] {}, + *absSystemTime); + } else if (absSteadyTime) { + res = parkingLot.park_until( + futex, + waitMask, + [&] { return *futex == expected; }, + [] {}, + *absSteadyTime); + } else { + res = parkingLot.park( + futex, waitMask, [&] { return *futex == expected; }, [] {}); + } + switch (res) { + case ParkResult::Skip: + return FutexResult::VALUE_CHANGED; + case ParkResult::Unpark: + return FutexResult::AWOKEN; + case ParkResult::Timeout: + return FutexResult::TIMEDOUT; + } + + return FutexResult::INTERRUPTED; +} + +} // namespace + +///////////////////////////////// +// Futex<> overloads + +int futexWakeImpl( + const Futex* futex, + int count, + uint32_t wakeMask) { +#ifdef __linux__ + return nativeFutexWake(futex, count, wakeMask); +#else + return emulatedFutexWake(futex, count, wakeMask); +#endif +} + +int futexWakeImpl( + const Futex* futex, + int count, + uint32_t wakeMask) { + return emulatedFutexWake(futex, count, wakeMask); +} + +FutexResult futexWaitImpl( + const Futex* futex, + uint32_t expected, + system_clock::time_point const* absSystemTime, + steady_clock::time_point const* absSteadyTime, + uint32_t waitMask) { +#ifdef __linux__ + return nativeFutexWaitImpl( + futex, expected, absSystemTime, absSteadyTime, waitMask); +#else + return emulatedFutexWaitImpl( + futex, expected, absSystemTime, absSteadyTime, waitMask); +#endif +} + +FutexResult futexWaitImpl( + const Futex* futex, + uint32_t expected, + system_clock::time_point const* absSystemTime, + steady_clock::time_point const* absSteadyTime, + uint32_t waitMask) { + return emulatedFutexWaitImpl( + futex, expected, absSystemTime, absSteadyTime, waitMask); +} + +} // namespace detail +} // namespace folly diff --git a/third-party/folly/folly/detail/Futex.h b/third-party/folly/folly/detail/Futex.h new file mode 100644 index 000000000..987a1b895 --- /dev/null +++ b/third-party/folly/folly/detail/Futex.h @@ -0,0 +1,96 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace folly { +namespace detail { + +enum class FutexResult { + VALUE_CHANGED, /* futex value didn't match expected */ + AWOKEN, /* wakeup by matching futex wake, or spurious wakeup */ + INTERRUPTED, /* wakeup by interrupting signal */ + TIMEDOUT, /* wakeup by expiring deadline */ +}; + +/** + * Futex is an atomic 32 bit unsigned integer that provides access to the + * futex() syscall on that value. It is templated in such a way that it + * can interact properly with DeterministicSchedule testing. + * + * If you don't know how to use futex(), you probably shouldn't be using + * this class. Even if you do know how, you should have a good reason + * (and benchmarks to back you up). + * + * Because of the semantics of the futex syscall, the futex family of + * functions are available as free functions rather than member functions + */ +template