diff --git a/benchmark/Makefile b/benchmark/Makefile index 07c7699..5b36272 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -22,7 +22,7 @@ build/sparsepp: bench.cc Makefile $(CXX) -DSPARSEPP -I../../sparsepp bench.cc -o $@ build/phmap: bench.cc Makefile $(PHMAP_SRC)/phmap.h - $(CXX) -DPHMAP -I.. bench.cc /MD -o $@ + $(CXX) -DPHMAP -I.. -I$(ABSEIL) bench.cc /MD -o $@ /link /LIBPATH:$(ABSEIL)/build/lib ${ABSEIL_LIBS} build/phmap_flat: bench.cc Makefile $(PHMAP_SRC)/phmap.h $(CXX) -DPHMAP_FLAT -I.. bench.cc /MD -o $@ @@ -39,7 +39,7 @@ test: builddir progs -rm -f output #./build/stl_unordered_map $(SIZE) random >> output #./build/sparsepp $(SIZE) random >> output - #./build/abseil_flat $(SIZE) random >> output + ./build/abseil_flat $(SIZE) random >> output #./build/phmap_flat $(SIZE) random >> output ./build/phmap $(SIZE) random >> output ./build/abseil_parallel_flat $(SIZE) random >> output diff --git a/benchmark/bench.cc b/benchmark/bench.cc index 540ec3f..927626f 100644 --- a/benchmark/bench.cc +++ b/benchmark/bench.cc @@ -23,13 +23,28 @@ #include "absl/container/parallel_flat_hash_map.h" #define MAPNAME absl::parallel_flat_hash_map #define NMSP absl + #define MTX absl::Mutex #else + #if 1 + // use Abseil's mutex... faster + #include "absl/synchronization/mutex.h" + struct AbslMutex : protected absl::Mutex + { + void lock() { this->Lock(); } + void unlock() { this->Unlock(); } + }; + #define MTX AbslMutex //std::mutex + #else + #include + #define MTX std::mutex + #endif + #include "parallel_hashmap/phmap.h" #define MAPNAME phmap::parallel_flat_hash_map #define NMSP phmap #endif - #define MT_SUPPORT 0 + #define MT_SUPPORT 2 #if MT_SUPPORT == 1 // create the parallel_flat_hash_map without internal mutexes, for when // we programatically ensure that each thread uses different internal submaps @@ -44,7 +59,7 @@ // -------------------------------------------------------------------------- #define EXTRAARGS , NMSP::container_internal::hash_default_hash, \ NMSP::container_internal::hash_default_eq, \ - std::allocator>, 4, NMSP::Mutex + std::allocator>, 4, MTX #else #define EXTRAARGS #endif @@ -207,14 +222,13 @@ void _fill_random_inner_mt(int64_t cnt, HT &hash, RSU &rsu) auto thread_fn = [&hash, cnt, num_threads](int64_t thread_idx, RSU rsu) { #if MT_SUPPORT - typename HT::hasher hasher; // get hasher object from the hash table [greg] todo provide hash fn size_t modulo = hash.subcnt() / num_threads; // subcnt() returns the number of submaps for (int64_t i=0; i #include #include -#include #include #include #include @@ -2503,7 +2502,7 @@ inline size_t RandomSeed() // ---------------------------------------------------------------------------- template class RefSet, - class Mutex, + class Mtx_, class Policy, class Hash, class Eq, class Alloc> class parallel_hash_set { @@ -2545,40 +2544,14 @@ public: using key_arg = typename KeyArgImpl::template type; protected: - // -------------------------------------------------------------------- - // MutexLock with the additional set_mutex function, otherwise we could - // make the MutexLock from mutex.h a template and use that one. - // -------------------------------------------------------------------- - class PHMAP_SCOPED_LOCKABLE MutexLock_ { - public: - explicit MutexLock_(Mutex *mu) PHMAP_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { - if (this->mu_) - this->mu_->lock(); - } - - void set_mutex(Mutex *mu) PHMAP_NO_THREAD_SAFETY_ANALYSIS { - assert(mu && this->mu_ == nullptr); - this->mu_ = mu; - this->mu_->lock(); - } - - MutexLock_(const MutexLock_ &) = delete; // NOLINT(runtime/mutex) - MutexLock_(MutexLock_&&) = delete; // NOLINT(runtime/mutex) - MutexLock_& operator=(const MutexLock_&) = delete; - MutexLock_& operator=(MutexLock_&&) = delete; - - ~MutexLock_() PHMAP_UNLOCK_FUNCTION() { if (this->mu_) this->mu_->unlock(); } - - private: - Mutex *mu_; - }; + using Lockable = phmap::LockableImpl; // -------------------------------------------------------------------- - struct alignas(64) Inner : public Mutex + struct alignas(64) Inner : public Lockable { bool operator==(const Inner& o) const { - phmap::scoped_lock l(const_cast(*this), const_cast(o)); + phmap::scoped_lock l(const_cast(*this), const_cast(o)); return set_ == o.set_; } @@ -3014,7 +2987,7 @@ public: Inner& inner = sets_[subidx(hash)]; auto& set = inner.set_; - MutexLock_ m(&inner); + typename Lockable::UniqueLock m(inner); auto res = set.insert(std::move(node), hash); return { make_iterator(&inner, res.position), res.inserted, @@ -3039,7 +3012,7 @@ public: size_t hash = HashElement{hash_ref()}(key); Inner& inner = sets_[subidx(hash)]; auto& set = inner.set_; - MutexLock_ m(&inner); + typename Lockable::UniqueLock m(inner); return make_rv(&inner, set.emplace_decomposable(key, hash, std::forward(args)...)); } @@ -3085,7 +3058,7 @@ public: size_t hash = HashElement{hash_ref()}(PolicyTraits::key(slot)); Inner& inner = sets_[subidx(hash)]; auto& set = inner.set_; - MutexLock_ m(&inner); + typename Lockable::UniqueLock m(inner); typename EmbeddedSet::template InsertSlotWithHash f { inner, std::move(*slot), hash}; return make_rv(PolicyTraits::apply(f, elem)); @@ -3114,7 +3087,7 @@ public: auto hash = HashElement{hash_ref()}(key); Inner& inner = sets_[subidx(hash)]; auto& set = inner.set_; - MutexLock_ m(&inner); + typename Lockable::UniqueLock m(inner); return make_iterator(&inner, set.lazy_emplace(key, hash, std::forward(f))); } @@ -3133,10 +3106,12 @@ public: auto hash = HashElement{hash_ref()}(key); Inner& inner = sets_[subidx(hash)]; auto& set = inner.set_; - MutexLock_ m(&inner); + typename Lockable::UpgradeLock m(inner); auto it = set.find(key, hash); if (it == set.end()) return 0; + + typename Lockable::UpgradeToUnique unique(m); set.erase(it); return 1; } @@ -3179,21 +3154,20 @@ public: // If the element already exists in `this`, it is left unmodified in `src`. // -------------------------------------------------------------------- template - void merge(parallel_hash_set& src) { // NOLINT + void merge(parallel_hash_set& src) { // NOLINT assert(this != &src); if (this != &src) { for (size_t i=0; i l(sets_[i], src.sets_[i]); sets_[i].set_.merge(src.sets_[i].set_); } } } template - void merge(parallel_hash_set&& src) { + void merge(parallel_hash_set&& src) { merge(src); } @@ -3216,8 +3190,7 @@ public: using std::swap; for (size_t i=0; i l(sets_[i], that.sets_[i]); swap(sets_[i].set_, that.sets_[i].set_); } } @@ -3226,7 +3199,7 @@ public: size_t nn = n / num_tables; for (auto& inner : sets_) { - MutexLock_ m(&inner); + typename Lockable::UniqueLock m(inner); inner.set_.rehash(nn); } } @@ -3257,11 +3230,11 @@ public: template void prefetch(const key_arg& key) const { (void)key; -#if defined(__GNUC__) +#if 0 && defined(__GNUC__) size_t hash = HashElement{hash_ref()}(key); const Inner& inner = sets_[subidx(hash)]; const auto& set = inner.set_; - MutexLock_ m(const_cast(&inner)); + typename Lockable::UniqueLock m(inner); set.prefetch_hash(hash); #endif // __GNUC__ } @@ -3278,7 +3251,7 @@ public: iterator find(const key_arg& key, size_t hash) { Inner& inner = sets_[subidx(hash)]; auto& set = inner.set_; - MutexLock_ m(&inner); + typename Lockable::SharedLock m(inner); auto it = set.find(key, hash); return make_iterator(&inner, it); } @@ -3322,7 +3295,7 @@ public: size_t sz = 0; for (const auto& inner : sets_) { - MutexLock_ m(const_cast(&inner)); + typename Lockable::SharedLock m(const_cast(inner)); sz += inner.set_.bucket_count(); } return sz; @@ -3402,7 +3375,7 @@ private: void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { for (auto& inner : sets_) { - MutexLock_ m(&inner); + typename Lockable::UniqueLock m(inner); inner.set_.drop_deletes_without_resize(); } } @@ -3411,7 +3384,7 @@ private: size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, elem); Inner& inner = sets_[subidx(hash)]; auto& set = inner.set_; - MutexLock_ m(const_cast(&inner)); + typename Lockable::SharedLock m(const_cast(inner)); return set.has_element(elem, hash); } @@ -3432,11 +3405,11 @@ private: protected: template std::tuple - find_or_prepare_insert(const K& key, MutexLock_ &mutexlock) { + find_or_prepare_insert(const K& key, typename Lockable::UniqueLock &mutexlock) { auto hash = HashElement{hash_ref()}(key); Inner& inner = sets_[subidx(hash)]; auto& set = inner.set_; - mutexlock.set_mutex(&inner); + mutexlock = std::move(typename Lockable::UniqueLock(inner)); auto p = set.find_or_prepare_insert(key, hash); // std::pair return std::make_tuple(&inner, p.first, p.second); } @@ -3454,6 +3427,11 @@ protected: return (hashval ^ (hashval >> N)) & mask; } + template + size_t hash(const K& key) { + return HashElement{hash_ref()}(key); + } + static size_t subcnt() { return num_tables; } @@ -3484,9 +3462,9 @@ private: // -------------------------------------------------------------------------- template class RefSet, - class Mutex, + class Mtx_, class Policy, class Hash, class Eq, class Alloc> -class parallel_hash_map : public parallel_hash_set +class parallel_hash_map : public parallel_hash_set { // P is Policy. It's passed as a template argument to support maps that have // incomplete types as values, as in unordered_map. @@ -3504,6 +3482,7 @@ class parallel_hash_map : public parallel_hash_set::value && IsTransparent::value>; using Base = typename parallel_hash_map::parallel_hash_set; + using Lockable = phmap::LockableImpl; public: using key_type = typename Policy::key_type; @@ -3623,8 +3602,8 @@ public: private: template std::pair insert_or_assign_impl(K&& k, V&& v) { - typename Base::MutexLock_ mutexlock(nullptr); - auto res = this->find_or_prepare_insert(k, mutexlock); + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(k, m); typename Base::Inner *inner = std::get<0>(res); if (std::get<2>(res)) inner->set_.emplace_at(std::get<1>(res), std::forward(k), std::forward(v)); @@ -3636,8 +3615,8 @@ private: template std::pair try_emplace_impl(K&& k, Args&&... args) { - typename Base::MutexLock_ mutexlock(nullptr); - auto res = this->find_or_prepare_insert(k, mutexlock); + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(k, m); typename Base::Inner *inner = std::get<0>(res); if (std::get<2>(res)) inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, @@ -4577,10 +4556,10 @@ public: // ----------------------------------------------------------------------------- // phmap::parallel_flat_hash_set // ----------------------------------------------------------------------------- -template // default values in phmap_fwd_decl.h +template // default values in phmap_fwd_decl.h class parallel_flat_hash_set : public phmap::container_internal::parallel_hash_set< - N, phmap::container_internal::raw_hash_set, Mutex, + N, phmap::container_internal::raw_hash_set, Mtx_, phmap::container_internal::FlatHashSetPolicy, Hash, Eq, Alloc> { @@ -4589,6 +4568,7 @@ class parallel_flat_hash_set public: parallel_flat_hash_set() {} using Base::Base; + using Base::hash; using Base::subidx; using Base::subcnt; using Base::begin; @@ -4624,9 +4604,9 @@ public: // ----------------------------------------------------------------------------- // phmap::parallel_flat_hash_map - default values in phmap_fwd_decl.h // ----------------------------------------------------------------------------- -template +template class parallel_flat_hash_map : public phmap::container_internal::parallel_hash_map< - N, phmap::container_internal::raw_hash_set, Mutex, + N, phmap::container_internal::raw_hash_set, Mtx_, phmap::container_internal::FlatHashMapPolicy, Hash, Eq, Alloc> { @@ -4635,6 +4615,7 @@ class parallel_flat_hash_map : public phmap::container_internal::parallel_hash_m public: parallel_flat_hash_map() {} using Base::Base; + using Base::hash; using Base::subidx; using Base::subcnt; using Base::begin; @@ -4674,10 +4655,10 @@ public: // ----------------------------------------------------------------------------- // phmap::parallel_node_hash_set // ----------------------------------------------------------------------------- -template +template class parallel_node_hash_set : public phmap::container_internal::parallel_hash_set< - N, phmap::container_internal::raw_hash_set, Mutex, + N, phmap::container_internal::raw_hash_set, Mtx_, phmap::container_internal::NodeHashSetPolicy, Hash, Eq, Alloc> { using Base = typename parallel_node_hash_set::parallel_hash_set; @@ -4685,6 +4666,9 @@ class parallel_node_hash_set public: parallel_node_hash_set() {} using Base::Base; + using Base::hash; + using Base::subidx; + using Base::subcnt; using Base::begin; using Base::cbegin; using Base::cend; @@ -4720,10 +4704,10 @@ public: // ----------------------------------------------------------------------------- // phmap::parallel_node_hash_map // ----------------------------------------------------------------------------- -template +template class parallel_node_hash_map : public phmap::container_internal::parallel_hash_map< - N, phmap::container_internal::raw_hash_set, Mutex, + N, phmap::container_internal::raw_hash_set, Mtx_, phmap::container_internal::NodeHashMapPolicy, Hash, Eq, Alloc> { @@ -4732,6 +4716,9 @@ class parallel_node_hash_map public: parallel_node_hash_map() {} using Base::Base; + using Base::hash; + using Base::subidx; + using Base::subcnt; using Base::begin; using Base::cbegin; using Base::cend; diff --git a/parallel_hashmap/phmap_base.h b/parallel_hashmap/phmap_base.h index 9b438ef..9d897c3 100644 --- a/parallel_hashmap/phmap_base.h +++ b/parallel_hashmap/phmap_base.h @@ -46,6 +46,7 @@ #include #include #include +#include // for std::lock #include "phmap_config.h" @@ -4506,6 +4507,20 @@ inline T& ts_unchecked_read(T& v) PHMAP_NO_THREAD_SAFETY_ANALYSIS { namespace phmap { +#if defined(BOOST_THREAD_SHARED_MUTEX_HPP) && defined(BOOST_THREAD_LOCK_TYPES_HPP) + #define PHMAP_HAS_BOOST_THREAD_MUTEXES +#endif + +#ifdef PHMAP_HAS_BOOST_THREAD_MUTEXES + using defer_lock_t = boost::defer_lock_t; + using try_to_lock_t = boost::try_to_lock_t; + using adopt_lock_t = boost::adopt_lock_t; +#else + struct adopt_lock_t { explicit adopt_lock_t() = default; }; + struct defer_lock_t { explicit defer_lock_t() = default; }; + struct try_to_lock_t { explicit try_to_lock_t() = default; }; +#endif + // ----------------------------------------------------------------------------- // NullMutex // ----------------------------------------------------------------------------- @@ -4522,16 +4537,12 @@ public: bool try_lock() { return true; } }; -struct adopt_lock_t { explicit adopt_lock_t() = default; }; -struct defer_lock_t { explicit defer_lock_t() = default; }; -struct try_to_lock_t { explicit try_to_lock_t() = default; }; - // --------------------------- simplified scoped_lock ------------------------------ template class scoped_lock { public: - using mutex_type = MutexType; // If MutexTypes... consists of the single type Mutex + using mutex_type = MutexType; explicit scoped_lock(mutex_type& m1, mutex_type& m2) : _m1(m1), _m2(m2) @@ -4564,14 +4575,14 @@ class LockableBaseImpl public: struct DoNothing { - DoNothing() {} + DoNothing() noexcept {} explicit DoNothing(Mutex& ) noexcept {} DoNothing(Mutex&, phmap::adopt_lock_t) noexcept {} DoNothing(Mutex&, phmap::defer_lock_t) noexcept {} DoNothing(Mutex&, phmap::try_to_lock_t) {} - void lock() {} - void unlock() {} - bool try_lock() { return true; } + template explicit DoNothing(T&) {} + template explicit DoNothing(T&&) {} + DoNothing& operator=(DoNothing&&) { return *this; } void swap(DoNothing &) {} bool owns_lock() const noexcept { return true; } }; @@ -4586,8 +4597,11 @@ public: {} explicit ScopedLock(Mutex &m) : - m_(&m), locked_(false) - { lock(m); } + m_(&m) + { + m_->lock(); + locked_ = true; + } ScopedLock(Mutex& m, adopt_lock_t) noexcept : m_(&m), locked_(true) @@ -4599,15 +4613,45 @@ public: ScopedLock(Mutex& m, try_to_lock_t) : m_(&m), locked_(false) - { try_lock(m); } + { + try_lock(); + } - ~ScopedLock() { if (locked_) m_->unlock(); } + ScopedLock(ScopedLock &&o) : + m_(std::move(o.m_)), locked_(std::move(o.locked_)) + { + o.locked_ = false; + o.m_ = nullptr; + } + + ScopedLock& operator=(ScopedLock&& other) + { + ScopedLock temp(std::move(other)); + swap(temp); + return *this; + } + + ~ScopedLock() + { + if (locked_) + m_->unlock(); + } void lock() - { if (!locked_) { m_->lock(); locked_ = true; } } + { + if (!locked_) { + m_->lock(); + locked_ = true; + } + } void unlock() - { if (locked_) { m_->unlock(); locked_ = false; } } + { + if (locked_) { + m_->unlock(); + locked_ = false; + } + } bool try_lock() { @@ -4648,12 +4692,12 @@ public: // } // // --------------------------------------------------------------------------- -template -class LockableImpl : public Mutex, public LockableBaseImpl +template +class LockableImpl : public Mtx_, public LockableBaseImpl { public: - using mutex_type = Mutex; - using Base = LockableBaseImpl; + using mutex_type = Mtx_; + using Base = LockableBaseImpl; using SharedLock = typename Base::ScopedLock; using UpgradeLock = typename Base::ScopedLock; using UniqueLock = typename Base::ScopedLock; @@ -4662,7 +4706,7 @@ public: // --------------------------------------------------------------------------- template <> -class LockableImpl: public LockableBaseImpl +class LockableImpl: public phmap::NullMutex, public LockableBaseImpl { public: using mutex_type = phmap::NullMutex; @@ -4673,7 +4717,7 @@ public: using UpgradeToUnique = typename Base::DoNothing; }; -#if defined(BOOST_THREAD_SHARED_MUTEX_HPP) && defined(BOOST_THREAD_LOCK_TYPES_HPP) +#ifdef PHMAP_HAS_BOOST_THREAD_MUTEXES // --------------------------------------------------------------------------- template <> @@ -4682,7 +4726,7 @@ class LockableImpl : public boost::shared_mutex public: using mutex_type = boost::shared_mutex; using SharedLock = boost::shared_lock; - using UpgradeLock = boost::unique_lock; // we assume that shared can't upgrade + using UpgradeLock = boost::unique_lock; // we assume that boost::shared_mutex can't upgrade using UniqueLock = boost::unique_lock; using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership }; @@ -4693,9 +4737,9 @@ class LockableImpl : public boost::upgrade_mutex { public: using mutex_type = boost::upgrade_mutex; - using UniqueLock = boost::unique_lock; using SharedLock = boost::shared_lock; using UpgradeLock = boost::upgrade_lock; + using UniqueLock = boost::unique_lock; using UpgradeToUnique = boost::upgrade_to_unique_lock; };