From 0cd5c6a64cac2c68c4424f8112a18b352e3c7cc0 Mon Sep 17 00:00:00 2001 From: greg Date: Sun, 17 Mar 2019 15:09:19 -0400 Subject: [PATCH] wip --- parallel_hashmap/phmap.h | 481 +++++++++++++++++++++++++++++++++- parallel_hashmap/phmap_base.h | 278 ++++++++++++++++++++ 2 files changed, 749 insertions(+), 10 deletions(-) diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 399b5e6..2683831 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -473,7 +473,7 @@ inline size_t GrowthToLowerboundCapacity(size_t growth) return growth + static_cast((static_cast(growth) - 1) / 7); } -namespace hashtable_debug_internal { +namespace debug { // If it is a map, call get<0>(). using std::get; @@ -516,7 +516,7 @@ struct HashtableDebugAccess } }; -} // namespace hashtable_debug_internal +} // namespace debug // ---------------------------------------------------------------------------- // I N F O Z S T U B S @@ -1520,8 +1520,7 @@ public: private: template - friend struct phmap::container_internal::hashtable_debug_internal:: - HashtableDebugAccess; + friend struct phmap::container_internal::debug::HashtableDebugAccess; struct FindElement { @@ -2196,14 +2195,14 @@ protected: // MutexLock with the additional set_mutex function, otherwise we could // make the MutexLock from mutex.h a template and use that one. // -------------------------------------------------------------------- - class SCOPED_LOCKABLE MutexLock_ { + class PHMAP_SCOPED_LOCKABLE MutexLock_ { public: - explicit MutexLock_(Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { + explicit MutexLock_(Mutex *mu) PHMAP_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { if (this->mu_) this->mu_->Lock(); } - void set_mutex(Mutex *mu) NO_THREAD_SAFETY_ANALYSIS { + void set_mutex(Mutex *mu) PHMAP_NO_THREAD_SAFETY_ANALYSIS { assert(mu && this->mu_ == nullptr); this->mu_ = mu; this->mu_->Lock(); @@ -2214,7 +2213,7 @@ protected: MutexLock_& operator=(const MutexLock_&) = delete; MutexLock_& operator=(MutexLock_&&) = delete; - ~MutexLock_() UNLOCK_FUNCTION() { if (this->mu_) this->mu_->Unlock(); } + ~MutexLock_() PHMAP_UNLOCK_FUNCTION() { if (this->mu_) this->mu_->Unlock(); } private: Mutex * mu_; @@ -2998,8 +2997,7 @@ public: private: template - friend struct phmap::container_internal::hashtable_debug_internal:: - HashtableDebugAccess; + friend struct phmap::container_internal::debug::HashtableDebugAccess; struct FindElement { @@ -3298,6 +3296,429 @@ private: } }; +// ---------------------------------------------------------------------------- +// Allocates at least n bytes aligned to the specified alignment. +// Alignment must be a power of 2. It must be positive. +// +// Note that many allocators don't honor alignment requirements above certain +// threshold (usually either alignof(std::max_align_t) or alignof(void*)). +// Allocate() doesn't apply alignment corrections. If the underlying allocator +// returns insufficiently alignment pointer, that's what you are going to get. +// ---------------------------------------------------------------------------- +template +void* Allocate(Alloc* alloc, size_t n) { + static_assert(Alignment > 0, ""); + assert(n && "n must be positive"); + struct alignas(Alignment) M {}; + using A = typename phmap::allocator_traits::template rebind_alloc; + using AT = typename phmap::allocator_traits::template rebind_traits; + A mem_alloc(*alloc); + void* p = AT::allocate(mem_alloc, (n + sizeof(M) - 1) / sizeof(M)); + assert(reinterpret_cast(p) % Alignment == 0 && + "allocator does not respect alignment"); + return p; +} + +// ---------------------------------------------------------------------------- +// The pointer must have been previously obtained by calling +// Allocate(alloc, n). +// ---------------------------------------------------------------------------- +template +void Deallocate(Alloc* alloc, void* p, size_t n) { + static_assert(Alignment > 0, ""); + assert(n && "n must be positive"); + struct alignas(Alignment) M {}; + using A = typename phmap::allocator_traits::template rebind_alloc; + using AT = typename phmap::allocator_traits::template rebind_traits; + A mem_alloc(*alloc); + AT::deallocate(mem_alloc, static_cast(p), + (n + sizeof(M) - 1) / sizeof(M)); +} + +namespace memory_internal { + +// Constructs T into uninitialized storage pointed by `ptr` using the args +// specified in the tuple. +// ---------------------------------------------------------------------------- +template +void ConstructFromTupleImpl(Alloc* alloc, T* ptr, Tuple&& t, + phmap::index_sequence) { + phmap::allocator_traits::construct( + *alloc, ptr, std::get(std::forward(t))...); +} + +template +struct WithConstructedImplF { + template + decltype(std::declval()(std::declval())) operator()( + Args&&... args) const { + return std::forward(f)(T(std::forward(args)...)); + } + F&& f; +}; + +template +decltype(std::declval()(std::declval())) WithConstructedImpl( + Tuple&& t, phmap::index_sequence, F&& f) { + return WithConstructedImplF{std::forward(f)}( + std::get(std::forward(t))...); +} + +template +auto TupleRefImpl(T&& t, phmap::index_sequence) + -> decltype(std::forward_as_tuple(std::get(std::forward(t))...)) { + return std::forward_as_tuple(std::get(std::forward(t))...); +} + +// Returns a tuple of references to the elements of the input tuple. T must be a +// tuple. +// ---------------------------------------------------------------------------- +template +auto TupleRef(T&& t) -> decltype( + TupleRefImpl(std::forward(t), + phmap::make_index_sequence< + std::tuple_size::type>::value>())) { + return TupleRefImpl( + std::forward(t), + phmap::make_index_sequence< + std::tuple_size::type>::value>()); +} + +template +decltype(std::declval()(std::declval(), std::piecewise_construct, + std::declval>(), std::declval())) +DecomposePairImpl(F&& f, std::pair, V> p) { + const auto& key = std::get<0>(p.first); + return std::forward(f)(key, std::piecewise_construct, std::move(p.first), + std::move(p.second)); +} + +} // namespace memory_internal + +// Constructs T into uninitialized storage pointed by `ptr` using the args +// specified in the tuple. +// ---------------------------------------------------------------------------- +template +void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) { + memory_internal::ConstructFromTupleImpl( + alloc, ptr, std::forward(t), + phmap::make_index_sequence< + std::tuple_size::type>::value>()); +} + +// Constructs T using the args specified in the tuple and calls F with the +// constructed value. +// ---------------------------------------------------------------------------- +template +decltype(std::declval()(std::declval())) WithConstructed( + Tuple&& t, F&& f) { + return memory_internal::WithConstructedImpl( + std::forward(t), + phmap::make_index_sequence< + std::tuple_size::type>::value>(), + std::forward(f)); +} + +// ---------------------------------------------------------------------------- +// Given arguments of an std::pair's consructor, PairArgs() returns a pair of +// tuples with references to the passed arguments. The tuples contain +// constructor arguments for the first and the second elements of the pair. +// +// The following two snippets are equivalent. +// +// 1. std::pair p(args...); +// +// 2. auto a = PairArgs(args...); +// std::pair p(std::piecewise_construct, +// std::move(p.first), std::move(p.second)); +// ---------------------------------------------------------------------------- +inline std::pair, std::tuple<>> PairArgs() { return {}; } + +template +std::pair, std::tuple> PairArgs(F&& f, S&& s) { + return {std::piecewise_construct, std::forward_as_tuple(std::forward(f)), + std::forward_as_tuple(std::forward(s))}; +} + +template +std::pair, std::tuple> PairArgs( + const std::pair& p) { + return PairArgs(p.first, p.second); +} + +template +std::pair, std::tuple> PairArgs(std::pair&& p) { + return PairArgs(std::forward(p.first), std::forward(p.second)); +} + +template +auto PairArgs(std::piecewise_construct_t, F&& f, S&& s) + -> decltype(std::make_pair(memory_internal::TupleRef(std::forward(f)), + memory_internal::TupleRef(std::forward(s)))) { + return std::make_pair(memory_internal::TupleRef(std::forward(f)), + memory_internal::TupleRef(std::forward(s))); +} + +// A helper function for implementing apply() in map policies. +// ---------------------------------------------------------------------------- +template +auto DecomposePair(F&& f, Args&&... args) + -> decltype(memory_internal::DecomposePairImpl( + std::forward(f), PairArgs(std::forward(args)...))) { + return memory_internal::DecomposePairImpl( + std::forward(f), PairArgs(std::forward(args)...)); +} + +// A helper function for implementing apply() in set policies. +// ---------------------------------------------------------------------------- +template +decltype(std::declval()(std::declval(), std::declval())) +DecomposeValue(F&& f, Arg&& arg) { + const auto& key = arg; + return std::forward(f)(key, std::forward(arg)); +} + +// Helper functions for asan and msan. +// ---------------------------------------------------------------------------- +inline void SanitizerPoisonMemoryRegion(const void* m, size_t s) { +#ifdef ADDRESS_SANITIZER + ASAN_POISON_MEMORY_REGION(m, s); +#endif +#ifdef MEMORY_SANITIZER + __msan_poison(m, s); +#endif + (void)m; + (void)s; +} + +inline void SanitizerUnpoisonMemoryRegion(const void* m, size_t s) { +#ifdef ADDRESS_SANITIZER + ASAN_UNPOISON_MEMORY_REGION(m, s); +#endif +#ifdef MEMORY_SANITIZER + __msan_unpoison(m, s); +#endif + (void)m; + (void)s; +} + +template +inline void SanitizerPoisonObject(const T* object) { + SanitizerPoisonMemoryRegion(object, sizeof(T)); +} + +template +inline void SanitizerUnpoisonObject(const T* object) { + SanitizerUnpoisonMemoryRegion(object, sizeof(T)); +} + +namespace memory_internal { + +// ---------------------------------------------------------------------------- +// If Pair is a standard-layout type, OffsetOf::kFirst and +// OffsetOf::kSecond are equivalent to offsetof(Pair, first) and +// offsetof(Pair, second) respectively. Otherwise they are -1. +// +// The purpose of OffsetOf is to avoid calling offsetof() on non-standard-layout +// type, which is non-portable. +// ---------------------------------------------------------------------------- +template +struct OffsetOf { + static constexpr size_t kFirst = -1; + static constexpr size_t kSecond = -1; +}; + +template +struct OffsetOf::type> +{ + static constexpr size_t kFirst = offsetof(Pair, first); + static constexpr size_t kSecond = offsetof(Pair, second); +}; + +// ---------------------------------------------------------------------------- +template +struct IsLayoutCompatible +{ +private: + struct Pair { + K first; + V second; + }; + + // Is P layout-compatible with Pair? + template + static constexpr bool LayoutCompatible() { + return std::is_standard_layout

() && sizeof(P) == sizeof(Pair) && + alignof(P) == alignof(Pair) && + memory_internal::OffsetOf

::kFirst == + memory_internal::OffsetOf::kFirst && + memory_internal::OffsetOf

::kSecond == + memory_internal::OffsetOf::kSecond; + } + +public: + // Whether pair and pair are layout-compatible. If they are, + // then it is safe to store them in a union and read from either. + static constexpr bool value = std::is_standard_layout() && + std::is_standard_layout() && + memory_internal::OffsetOf::kFirst == 0 && + LayoutCompatible>() && + LayoutCompatible>(); +}; + +} // namespace memory_internal + +// ---------------------------------------------------------------------------- +// The internal storage type for key-value containers like flat_hash_map. +// +// It is convenient for the value_type of a flat_hash_map to be +// pair; the "const K" prevents accidental modification of the key +// when dealing with the reference returned from find() and similar methods. +// However, this creates other problems; we want to be able to emplace(K, V) +// efficiently with move operations, and similarly be able to move a +// pair in insert(). +// +// The solution is this union, which aliases the const and non-const versions +// of the pair. This also allows flat_hash_map to work, even though +// that has the same efficiency issues with move in emplace() and insert() - +// but people do it anyway. +// +// If kMutableKeys is false, only the value member can be accessed. +// +// If kMutableKeys is true, key can be accessed through all slots while value +// and mutable_value must be accessed only via INITIALIZED slots. Slots are +// created and destroyed via mutable_value so that the key can be moved later. +// +// Accessing one of the union fields while the other is active is safe as +// long as they are layout-compatible, which is guaranteed by the definition of +// kMutableKeys. For C++11, the relevant section of the standard is +// https://timsong-cpp.github.io/cppwp/n3337/class.mem#19 (9.2.19) +// ---------------------------------------------------------------------------- +template +union map_slot_type +{ + map_slot_type() {} + ~map_slot_type() = delete; + using value_type = std::pair; + using mutable_value_type = std::pair; + + value_type value; + mutable_value_type mutable_value; + K key; +}; + +// ---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- +template +struct map_slot_policy +{ + using slot_type = map_slot_type; + using value_type = std::pair; + using mutable_value_type = std::pair; + +private: + static void emplace(slot_type* slot) { + // The construction of union doesn't do anything at runtime but it allows us + // to access its members without violating aliasing rules. + new (slot) slot_type; + } + // If pair and pair are layout-compatible, we can accept one + // or the other via slot_type. We are also free to access the key via + // slot_type::key in this case. + using kMutableKeys = memory_internal::IsLayoutCompatible; + +public: + static value_type& element(slot_type* slot) { return slot->value; } + static const value_type& element(const slot_type* slot) { + return slot->value; + } + + static const K& key(const slot_type* slot) { + return kMutableKeys::value ? slot->key : slot->value.first; + } + + template + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + emplace(slot); + if (kMutableKeys::value) { + phmap::allocator_traits::construct(*alloc, &slot->mutable_value, + std::forward(args)...); + } else { + phmap::allocator_traits::construct(*alloc, &slot->value, + std::forward(args)...); + } + } + + // Construct this slot by moving from another slot. + template + static void construct(Allocator* alloc, slot_type* slot, slot_type* other) { + emplace(slot); + if (kMutableKeys::value) { + phmap::allocator_traits::construct( + *alloc, &slot->mutable_value, std::move(other->mutable_value)); + } else { + phmap::allocator_traits::construct(*alloc, &slot->value, + std::move(other->value)); + } + } + + template + static void destroy(Allocator* alloc, slot_type* slot) { + if (kMutableKeys::value) { + phmap::allocator_traits::destroy(*alloc, &slot->mutable_value); + } else { + phmap::allocator_traits::destroy(*alloc, &slot->value); + } + } + + template + static void transfer(Allocator* alloc, slot_type* new_slot, + slot_type* old_slot) { + emplace(new_slot); + if (kMutableKeys::value) { + phmap::allocator_traits::construct( + *alloc, &new_slot->mutable_value, std::move(old_slot->mutable_value)); + } else { + phmap::allocator_traits::construct(*alloc, &new_slot->value, + std::move(old_slot->value)); + } + destroy(alloc, old_slot); + } + + template + static void swap(Allocator* alloc, slot_type* a, slot_type* b) { + if (kMutableKeys::value) { + using std::swap; + swap(a->mutable_value, b->mutable_value); + } else { + value_type tmp = std::move(a->value); + phmap::allocator_traits::destroy(*alloc, &a->value); + phmap::allocator_traits::construct(*alloc, &a->value, + std::move(b->value)); + phmap::allocator_traits::destroy(*alloc, &b->value); + phmap::allocator_traits::construct(*alloc, &b->value, + std::move(tmp)); + } + } + + template + static void move(Allocator* alloc, slot_type* src, slot_type* dest) { + if (kMutableKeys::value) { + dest->mutable_value = std::move(src->mutable_value); + } else { + phmap::allocator_traits::destroy(*alloc, &dest->value); + phmap::allocator_traits::construct(*alloc, &dest->value, + std::move(src->value)); + } + } + + template + static void move(Allocator* alloc, slot_type* first, slot_type* last, + slot_type* result) { + for (slot_type *src = first, *dest = result; src != last; ++src, ++dest) + move(alloc, src, dest); + } +}; // -------------------------------------------------------------------------- // Policy: a policy defines how to perform different operations on @@ -3400,6 +3821,46 @@ struct FlatHashMapPolicy static const V& value(const std::pair* kv) { return kv->second; } }; +template +struct node_hash_policy { + static_assert(std::is_lvalue_reference::value, ""); + + using slot_type = typename std::remove_cv< + typename std::remove_reference::type>::type*; + + template + static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { + *slot = Policy::new_element(alloc, std::forward(args)...); + } + + template + static void destroy(Alloc* alloc, slot_type* slot) { + Policy::delete_element(alloc, *slot); + } + + template + static void transfer(Alloc*, slot_type* new_slot, slot_type* old_slot) { + *new_slot = *old_slot; + } + + static size_t space_used(const slot_type* slot) { + if (slot == nullptr) return Policy::element_space_used(nullptr); + return Policy::element_space_used(*slot); + } + + static Reference element(slot_type* slot) { return **slot; } + + template + static auto value(T* elem) -> decltype(P::value(elem)) { + return P::value(elem); + } + + template + static auto apply(Ts&&... ts) -> decltype(P::apply(std::forward(ts)...)) { + return P::apply(std::forward(ts)...); + } +}; + // -------------------------------------------------------------------------- // -------------------------------------------------------------------------- template diff --git a/parallel_hashmap/phmap_base.h b/parallel_hashmap/phmap_base.h index e728e71..aede512 100644 --- a/parallel_hashmap/phmap_base.h +++ b/parallel_hashmap/phmap_base.h @@ -3770,6 +3770,7 @@ template using EnableIf = typename std::enable_if::type; // Can `T` be a template argument of `Layout`? +// --------------------------------------------------------------------------- template using IsLegalElementType = std::integral_constant< bool, !std::is_reference::value && !std::is_volatile::value && @@ -3780,6 +3781,7 @@ using IsLegalElementType = std::integral_constant< template class LayoutImpl; +// --------------------------------------------------------------------------- // Public base class of `Layout` and the result type of `Layout::Partial()`. // // `Elements...` contains all template arguments of `Layout` that created this @@ -3791,6 +3793,7 @@ class LayoutImpl; // `OffsetSeq...` is `[0, NumOffsets)` where `NumOffsets` is // `Min(sizeof...(Elements), NumSizes + 1)` (the number of arrays for which we // can compute offsets). +// --------------------------------------------------------------------------- template class LayoutImpl, phmap::index_sequence, phmap::index_sequence> @@ -4098,6 +4101,7 @@ public: // be missing (as in the example above). Only fields with known offsets are // described. Type names may differ across platforms: one compiler might // produce "unsigned*" where another produces "unsigned int *". + // --------------------------------------------------------------------------- std::string DebugString() const { const auto offsets = Offsets(); const size_t sizes[] = {SizeOf>()...}; @@ -4129,12 +4133,14 @@ using LayoutType = LayoutImpl< } // namespace internal_layout +// --------------------------------------------------------------------------- // Descriptor of arrays of various types and sizes laid out in memory one after // another. See the top of the file for documentation. // // Check out the public API of internal_layout::LayoutImpl above. The type is // internal to the library but its methods are public, and they are inherited // by `Layout`. +// --------------------------------------------------------------------------- template class Layout : public internal_layout::LayoutType { @@ -4168,5 +4174,277 @@ public: } // namespace container_internal } // namespace phmap +// --------------------------------------------------------------------------- +// compressed_tuple.h +// --------------------------------------------------------------------------- + +#ifdef _MSC_VER + // We need to mark these classes with this declspec to ensure that + // CompressedTuple happens. + #define PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC __declspec(empty_bases) +#else // _MSC_VER + #define PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC +#endif // _MSC_VER + +namespace phmap { +namespace container_internal { + +template +class CompressedTuple; + +namespace internal_compressed_tuple { + +template +struct Elem; +template +struct Elem, I> + : std::tuple_element> {}; +template +using ElemT = typename Elem::type; + +// --------------------------------------------------------------------------- +// Use the __is_final intrinsic if available. Where it's not available, classes +// declared with the 'final' specifier cannot be used as CompressedTuple +// elements. +// TODO(sbenza): Replace this with std::is_final in C++14. +// --------------------------------------------------------------------------- +template +constexpr bool IsFinal() { +#if defined(__clang__) || defined(__GNUC__) + return __is_final(T); +#else + return false; +#endif +} + +template +constexpr bool ShouldUseBase() { + return std::is_class::value && std::is_empty::value && !IsFinal(); +} + +// The storage class provides two specializations: +// - For empty classes, it stores T as a base class. +// - For everything else, it stores T as a member. +// ------------------------------------------------ +template >()> +struct Storage +{ + using T = ElemT; + T value; + constexpr Storage() = default; + explicit constexpr Storage(T&& v) : value(phmap::forward(v)) {} + constexpr const T& get() const& { return value; } + T& get() & { return value; } + constexpr const T&& get() const&& { return phmap::move(*this).value; } + T&& get() && { return std::move(*this).value; } +}; + +template +struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC Storage + : ElemT +{ + using T = internal_compressed_tuple::ElemT; + constexpr Storage() = default; + explicit constexpr Storage(T&& v) : T(phmap::forward(v)) {} + constexpr const T& get() const& { return *this; } + T& get() & { return *this; } + constexpr const T&& get() const&& { return phmap::move(*this); } + T&& get() && { return std::move(*this); } +}; + +template +struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl; + +template +struct PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC + CompressedTupleImpl, phmap::index_sequence> + // We use the dummy identity function through std::integral_constant to + // convince MSVC of accepting and expanding I in that context. Without it + // you would get: + // error C3548: 'I': parameter pack cannot be used in this context + : Storage, + std::integral_constant::value>... +{ + constexpr CompressedTupleImpl() = default; + explicit constexpr CompressedTupleImpl(Ts&&... args) + : Storage, I>(phmap::forward(args))... {} +}; + +} // namespace internal_compressed_tuple + +// --------------------------------------------------------------------------- +// Helper class to perform the Empty Base Class Optimization. +// Ts can contain classes and non-classes, empty or not. For the ones that +// are empty classes, we perform the CompressedTuple. If all types in Ts are +// empty classes, then CompressedTuple is itself an empty class. +// +// To access the members, use member .get() function. +// +// Eg: +// phmap::container_internal::CompressedTuple value(7, t1, t2, +// t3); +// assert(value.get<0>() == 7); +// T1& t1 = value.get<1>(); +// const T2& t2 = value.get<2>(); +// ... +// +// https://en.cppreference.com/w/cpp/language/ebo +// --------------------------------------------------------------------------- +template +class PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple + : private internal_compressed_tuple::CompressedTupleImpl< + CompressedTuple, phmap::index_sequence_for> +{ +private: + template + using ElemT = internal_compressed_tuple::ElemT; + +public: + constexpr CompressedTuple() = default; + explicit constexpr CompressedTuple(Ts... base) + : CompressedTuple::CompressedTupleImpl(phmap::forward(base)...) {} + + template + ElemT& get() & { + return internal_compressed_tuple::Storage::get(); + } + + template + constexpr const ElemT& get() const& { + return internal_compressed_tuple::Storage::get(); + } + + template + ElemT&& get() && { + return std::move(*this) + .internal_compressed_tuple::template Storage::get(); + } + + template + constexpr const ElemT&& get() const&& { + return phmap::move(*this) + .internal_compressed_tuple::template Storage::get(); + } +}; + +// Explicit specialization for a zero-element tuple +// (needed to avoid ambiguous overloads for the default constructor). +// --------------------------------------------------------------------------- +template <> +class PHMAP_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple<> {}; + +} // namespace container_internal +} // namespace phmap + +// --------------------------------------------------------------------------- +// thread_annotations.h +// --------------------------------------------------------------------------- + +#if defined(__clang__) + #define PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) +#else + #define PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op +#endif + +#define PHMAP_GUARDED_BY(x) PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) +#define PHMAP_PT_GUARDED_BY(x) PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) + +#define PHMAP_ACQUIRED_AFTER(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) + +#define PHMAP_ACQUIRED_BEFORE(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) + +#define PHMAP_EXCLUSIVE_LOCKS_REQUIRED(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(__VA_ARGS__)) + +#define PHMAP_SHARED_LOCKS_REQUIRED(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(__VA_ARGS__)) + +#define PHMAP_LOCKS_EXCLUDED(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) + +#define PHMAP_LOCK_RETURNED(x) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) + +#define PHMAP_LOCKABLE \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(lockable) + +#define PHMAP_SCOPED_LOCKABLE \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) + +#define PHMAP_EXCLUSIVE_LOCK_FUNCTION(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock_function(__VA_ARGS__)) + +#define PHMAP_SHARED_LOCK_FUNCTION(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_lock_function(__VA_ARGS__)) + +#define PHMAP_UNLOCK_FUNCTION(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__)) + +#define PHMAP_EXCLUSIVE_TRYLOCK_FUNCTION(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock_function(__VA_ARGS__)) + +#define PHMAP_SHARED_TRYLOCK_FUNCTION(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock_function(__VA_ARGS__)) + +#define PHMAP_ASSERT_EXCLUSIVE_LOCK(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(assert_exclusive_lock(__VA_ARGS__)) + +#define PHMAP_ASSERT_SHARED_LOCK(...) \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_lock(__VA_ARGS__)) + +#define PHMAP_NO_THREAD_SAFETY_ANALYSIS \ + PHMAP_THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) + +//------------------------------------------------------------------------------ +// Tool-Supplied Annotations +//------------------------------------------------------------------------------ + +// TS_UNCHECKED should be placed around lock expressions that are not valid +// C++ syntax, but which are present for documentation purposes. These +// annotations will be ignored by the analysis. +#define PHMAP_TS_UNCHECKED(x) "" + +// TS_FIXME is used to mark lock expressions that are not valid C++ syntax. +// It is used by automated tools to mark and disable invalid expressions. +// The annotation should either be fixed, or changed to TS_UNCHECKED. +#define PHMAP_TS_FIXME(x) "" + +// Like NO_THREAD_SAFETY_ANALYSIS, this turns off checking within the body of +// a particular function. However, this attribute is used to mark functions +// that are incorrect and need to be fixed. It is used by automated tools to +// avoid breaking the build when the analysis is updated. +// Code owners are expected to eventually fix the routine. +#define PHMAP_NO_THREAD_SAFETY_ANALYSIS_FIXME PHMAP_NO_THREAD_SAFETY_ANALYSIS + +// Similar to NO_THREAD_SAFETY_ANALYSIS_FIXME, this macro marks a GUARDED_BY +// annotation that needs to be fixed, because it is producing thread safety +// warning. It disables the GUARDED_BY. +#define PHMAP_GUARDED_BY_FIXME(x) + +// Disables warnings for a single read operation. This can be used to avoid +// warnings when it is known that the read is not actually involved in a race, +// but the compiler cannot confirm that. +#define PHMAP_TS_UNCHECKED_READ(x) thread_safety_analysis::ts_unchecked_read(x) + + +namespace phmap { +namespace thread_safety_analysis { + +// Takes a reference to a guarded data member, and returns an unguarded +// reference. +template +inline const T& ts_unchecked_read(const T& v) PHMAP_NO_THREAD_SAFETY_ANALYSIS { + return v; +} + +template +inline T& ts_unchecked_read(T& v) PHMAP_NO_THREAD_SAFETY_ANALYSIS { + return v; +} + +} // namespace thread_safety_analysis +} // phmap #endif // phmap_base_h_guard_