diff --git a/examples/dump_load.cc b/examples/dump_load.cc index 6369c0c..ec8cba7 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -1,6 +1,6 @@ #include #include -#include +#include using phmap::flat_hash_map; using phmap::parallel_flat_hash_map; @@ -21,7 +21,7 @@ void dump_load_uint64_uint32() { phmap::BinaryInputArchive ar_in("./dump.data"); mp2.load(ar_in); // Iterate and print keys and values g|++ - for (const auto& n : mp2) + for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; } @@ -49,10 +49,83 @@ void dump_load_parallel_flat_hash_map() { std::cout << "key: " << n.first << ", value: " << n.second << "\n"; } +#if defined(__linux__) +void mmap_load_uint64_uint32() { + using MapType = flat_hash_map, + phmap::container_internal::hash_default_eq, + phmap::MmapAllocator< + phmap::container_internal::Pair>>; + MapType mp1; + mp1.reserve(100); + phmap::MmapOutputArchive ar_out("./dump.data"); + // Add a new entry + mp1[100] = 99; + mp1[300] = 299; + + // Iterate and print keys and values + for (const auto& n : mp1) + std::cout << n.first << "'s value is: " << n.second << "\n"; + + mp1.mmap_dump(ar_out); + MapType mp2; + + phmap::MmapInputArchive ar_in("./dump.data"); + mp2.mmap_load(ar_in); + mp2[849242] = 141; + mp2[11] = 1111; + // Iterate and print keys and values g|++ + for (const auto& n : mp2) + std::cout << n.first << "'s value is: " << n.second << "\n"; +} + +void mmap_load_parallel_flat_hash_map() { + using MapType = parallel_flat_hash_map, + phmap::container_internal::hash_default_eq, + phmap::MmapAllocator< + phmap::container_internal::Pair>, + 4, + phmap::NullMutex>; + + MapType mp1; + phmap::MmapOutputArchive ar_out("./dump.data"); + + // Add a new entry + mp1[100] = 99; + mp1[300] = 299; + mp1[101] = 992; + mp1[1300] = 2991; + mp1[1130] = 299; + mp1[2130] = 1299; + // Iterate and print + for (const auto& n : mp1) + std::cout << "key: " << n.first << ", value: " << n.second << "\n"; + + mp1.mmap_dump(ar_out); + MapType mp2; + phmap::MmapInputArchive ar_in("./dump.data"); + + mp2.mmap_load(ar_in); + std::cout << "[debug] map capacity: " << mp2.capacity() << ", size: " << mp2.size() << std::endl; + for (size_t i = 0; i < 100; i ++) { + mp2[6771 + i] = i; + } + std::cout << "[debug] map capacity: " << mp2.capacity() << ", size: " << mp2.size() << std::endl; + for (const auto& n : mp2) + std::cout << "key: " << n.first << ", value: " << n.second << "\n"; +} +#endif + int main() { dump_load_uint64_uint32(); dump_load_parallel_flat_hash_map(); + +#if defined(__linux__) + mmap_load_uint64_uint32(); + mmap_load_parallel_flat_hash_map(); +#endif return 0; } diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 506e941..454a209 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -45,9 +45,6 @@ #include #include #include -#include -#include -#include #include "phmap_utils.h" #include "phmap_base.h" @@ -1537,62 +1534,17 @@ public: } } - template - typename std::enable_if::value, bool>::type - dump(OutputArchive& ar) { - typename OutputArchive::Guard guard(&ar); - if (!ar.dump(size_)) { - std::cerr << "Failed to dump size_" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.dump(capacity_)) { - std::cerr << "Failed to dump capacity_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { + template + bool dump(OutputArchive&); - std::cerr << "Failed to dump ctrl_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast(slots_), sizeof(slot_type) * capacity_)) { - std::cerr << "Failed to dump slot_" << std::endl; - return false; - } - return true; - } + template + bool load(InputArchive&); - template - typename std::enable_if::value, bool>::type - load(InputArchive& ar) { - typename InputArchive::Guard guard(&ar); - if (!ar.load(&size_)){ - std::cerr << "Failed to load size_" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.load(&capacity_)) { - std::cerr << "Failed to load capacity_" << std::endl; - return false; - } - // allocate memory for ctrl_ and slots_ - initialize_slots(); - if (!ar.load(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { - std::cerr << "Failed to load ctrl" << std::endl; - return false; - } - if (!ar.load(reinterpret_cast(slots_), sizeof(slot_type) * capacity_)) { - std::cerr << "Failed to load slot" << std::endl; - return false; - } - return true; - } + template + bool mmap_dump(OutputArchive&); + + template + bool mmap_load(MmapInputArchive&); void rehash(size_t n) { if (n == 0 && capacity_ == 0) return; @@ -3199,50 +3151,17 @@ public: a.swap(b); } - template - typename std::enable_if::value, bool>::type - dump(OutputArchive& ar) { - typename OutputArchive::Guard guard(&ar); - if (! ar.dump(subcnt())) { - std::cerr << "Failed to dump meta!" << std::endl; - return false; - } - for (size_t i = 0; i < sets_.size(); ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.dump(ar)) { - std::cerr << "Failed to dump submap " << i << std::endl; - return false; - } - } - return true; - } + template + bool dump(OutputArchive& ar); - template - typename std::enable_if::value, bool>::type - load(InputArchive& ar) { - typename InputArchive::Guard guard(&ar); - size_t submap_count = 0; - if (!ar.load(&submap_count)) { - std::cerr << "Failed to load submap count!" << std::endl; - return false; - } + template + bool load(InputArchive& ar); - if (submap_count != subcnt()) { - std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; - return false; - } + template + bool mmap_dump(OutputArchive& ar); - for (size_t i = 0; i < submap_count; ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.load(ar)) { - std::cerr << "Failed to load submap " << i << std::endl; - return false; - } - } - return true; - } + template + bool mmap_load(InputArchive& ar); private: template diff --git a/parallel_hashmap/phmap_base.h b/parallel_hashmap/phmap_base.h index b2534bd..bbc6712 100644 --- a/parallel_hashmap/phmap_base.h +++ b/parallel_hashmap/phmap_base.h @@ -33,6 +33,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // --------------------------------------------------------------------------- + #include #include #include @@ -70,35 +71,6 @@ struct EqualTo namespace type_traits_internal { -template -struct PairTrait : public std::false_type { - using first_type = typename std::remove_cv::type; - using second_type = typename std::remove_cv::type; -}; - -template -struct PairTrait>: public std::true_type { - using first_type = T1; - using second_type = T2; -}; - -template -#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801 -struct IsTriviallyCopyable : public std::integral_constant { -}; -#else -struct IsTriviallyCopyable : public std::is_trivially_copyable { -}; -#endif - -template -struct IsDumpableType { - static constexpr bool value = IsTriviallyCopyable::value - || (PairTrait::value - && IsTriviallyCopyable::first_type>::value - && IsTriviallyCopyable::second_type>::value); -}; - template struct VoidTImpl { using type = void; diff --git a/parallel_hashmap/phmap_dump.h b/parallel_hashmap/phmap_dump.h new file mode 100644 index 0000000..b75262a --- /dev/null +++ b/parallel_hashmap/phmap_dump.h @@ -0,0 +1,635 @@ +#if !defined(phmap_dump_h_guard_) +#define phmap_dump_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com +// +// providing dump/load/mmap_load +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +#include +#include +#include +#include +#include +#include "phmap.h" +namespace phmap +{ + +namespace type_traits_internal { + +#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801 + template struct IsTriviallyCopyable : public std::integral_constant {}; +#else + template struct IsTriviallyCopyable : public std::is_trivially_copyable {}; +#endif + +template +struct IsTriviallyCopyable> { + static constexpr bool value = IsTriviallyCopyable::value && IsTriviallyCopyable::value; +}; +} + +namespace container_internal { + +//// raw_hash_set +template +template +bool raw_hash_set::dump(OutputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename OutputArchive::Guard guard(&ar); + if (!ar.dump(size_)) { + std::cerr << "Failed to dump size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.dump(capacity_)) { + std::cerr << "Failed to dump capacity_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { + + std::cerr << "Failed to dump ctrl_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(slots_), + sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to dump slot_" << std::endl; + return false; + } + return true; +} + +template +template +bool raw_hash_set::load(InputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename InputArchive::Guard guard(&ar); + if (!ar.load(&size_)) { + std::cerr << "Failed to load size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.load(&capacity_)) { + std::cerr << "Failed to load capacity_" << std::endl; + return false; + } + + // allocate memory for ctrl_ and slots_ + initialize_slots(); + if (!ar.load(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { + std::cerr << "Failed to load ctrl" << std::endl; + return false; + } + if (!ar.load(reinterpret_cast(slots_), + sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to load slot" << std::endl; + return false; + } + return true; +} + + +template +template +bool raw_hash_set::mmap_dump(OutputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename OutputArchive::Guard guard(&ar); + size_t align_size = Layout::Alignment(); + if (!ar.dump(size_, align_size)) { + std::cerr << "Failed to dump size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.dump(capacity_, align_size)) { + std::cerr << "Failed to dump capacity_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1), align_size)) { + + std::cerr << "Failed to dump ctrl_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(slots_), + sizeof(slot_type) * capacity_, align_size)) { + std::cerr << "Failed to dump slot_" << std::endl; + return false; + } + return true; +} + +template +template +bool raw_hash_set::mmap_load(MmapInputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + assert(ar.initialized()); + auto closure = ar.closure(); + this->alloc_ref().set_closure(closure); + size_t align_size = Layout::Alignment(); + if (!ar.load(&size_, align_size)) { + std::cerr << "Failed to load size!" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.load(&capacity_, align_size)) { + std::cerr << "Failed to load capacity!" << std::endl; + return false; + } + + + if (std::is_same>::value) { + infoz_ = Sample(); + } + reset_growth_left(); + infoz_.RecordStorageChanged(size_, capacity_); + + char* p_ctrl = ar.load( + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1), align_size); + ctrl_ = reinterpret_cast(p_ctrl); + + char* p_slots = ar.load(sizeof(slot_type) * capacity_, align_size); + slots_ = reinterpret_cast(p_slots); + return true; +} + + +////// parallel_hash_set +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template +bool parallel_hash_set::dump(OutputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename OutputArchive::Guard guard(&ar); + if (! ar.dump(subcnt())) { + std::cerr << "Failed to dump meta!" << std::endl; + return false; + } + for (size_t i = 0; i < sets_.size(); ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.dump(ar)) { + std::cerr << "Failed to dump submap " << i << std::endl; + return false; + } + } + return true; +} + +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template +bool parallel_hash_set::load(InputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename InputArchive::Guard guard(&ar); + size_t submap_count = 0; + if (!ar.load(&submap_count)) { + std::cerr << "Failed to load submap count!" << std::endl; + return false; + } + + if (submap_count != subcnt()) { + std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; + return false; + } + + for (size_t i = 0; i < submap_count; ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.load(ar)) { + std::cerr << "Failed to load submap " << i << std::endl; + return false; + } + } + return true; +} + +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template +bool parallel_hash_set::mmap_dump(OutputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename OutputArchive::Guard guard(&ar); + size_t align_size = EmbeddedSet::Layout::Alignment(); + if (! ar.dump(subcnt(), align_size)) { + std::cerr << "Failed to dump meta!" << std::endl; + return false; + } + for (size_t i = 0; i < sets_.size(); ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.mmap_dump(ar)) { + std::cerr << "Failed to dump submap " << i << std::endl; + return false; + } + } + return true; +} + +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template +bool parallel_hash_set::mmap_load(InputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + assert(ar.initialized()); + auto closure = ar.closure(); + this->alloc_ref().set_closure(closure); + size_t submap_count = 0; + size_t align_size = EmbeddedSet::Layout::Alignment(); + if (!ar.load(&submap_count, align_size)) { + std::cerr << "Failed to load submap count!" << std::endl; + return false; + } + + if (submap_count != subcnt()) { + std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; + return false; + } + + for (size_t i = 0; i < submap_count; ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.mmap_load(ar)) { + std::cerr << "Failed to load submap " << i << std::endl; + return false; + } + } + return true; +} +} // namesapce container_internal + + + +// ArchiveOutput & ArchiveInput + +#define CHECK_FILE(f) { \ + if (!f.is_open()) { \ + std::cerr << "File is not open!" << std::endl; \ + return false; \ + } \ +} + +template +class ArchiveGuard { +public: + ArchiveGuard(Archive* ar): ar_(ar) { + if (ar_->guard_ == NULL) { + ar_->guard_ = this; + } + }; + ~ArchiveGuard() { + if (ar_ && ar_->guard_ == this) { + ar_->finish(); + } + } +private: + Archive* ar_; +}; + +class BinaryOutputArchive { +public: + using Guard = ArchiveGuard; + + BinaryOutputArchive(const std::string& file_path): offset_(0), guard_(NULL) { + ofs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~BinaryOutputArchive() { + finish(); + } + + bool dump(char* p, size_t sz) { + CHECK_FILE(ofs_); + ofs_.write(p, sz); + offset_ += sz; + return true; + } + + template + typename std::enable_if::value, bool>::type + dump(const V& v) { + CHECK_FILE(ofs_); + ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); + offset_ += sizeof(V); + return true; + } + + void finish() { + if (ofs_.is_open()) { + ofs_.close(); + offset_ = 0; + } + } + +private: + friend class ArchiveGuard; + std::ofstream ofs_; + size_t offset_; + Guard* guard_; +}; + + +class BinaryInputArchive { +public: + using Guard = ArchiveGuard; + + BinaryInputArchive(const std::string& file_path): offset_(0), guard_(NULL) { + ifs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~BinaryInputArchive() { + finish(); + } + + bool load(char* p, size_t sz) { + CHECK_FILE(ifs_); + ifs_.read(p, sz); + offset_ += sz; + return true; + } + + template + typename std::enable_if::value, bool>::type + load(V* v) { + CHECK_FILE(ifs_); + ifs_.read(reinterpret_cast(v), sizeof(V)); + offset_ += sizeof(V); + return true; + } + + void finish() { + if (ifs_.is_open()) { + ifs_.close(); + offset_ = 0; + } + } + +private: + friend class ArchiveGuard; + std::ifstream ifs_; + size_t offset_; + Guard* guard_; +}; + +#if defined(__linux__) // only support linux's mmap now +// mmap dump && mmap load +#include +#include +#include +#include + +class MmapOutputArchive { +public: + using Guard = ArchiveGuard; + + MmapOutputArchive(const std::string& file_path): offset_(0), guard_(NULL) { + ofs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~MmapOutputArchive() { + finish(); + } + + bool dump(char* p, size_t sz, size_t align_size) { + CHECK_FILE(ofs_); + ofs_.write(p, sz); + offset_ += sz; + return align(align_size); + } + + template + typename std::enable_if::value, bool>::type + dump(const V& v, size_t align_size) { + CHECK_FILE(ofs_); + ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); + offset_ += sizeof(V); + return align(align_size); + } + + void finish() { + if (ofs_.is_open()) { + ofs_.close(); + offset_ = 0; + } + } +private: + // padding for align + bool align(size_t align) { + size_t padding_size = (offset_ + align - 1) / align * align - offset_; + if (padding_size == 0) { + return true; + } + std::string padding(padding_size, '\0'); + ofs_.write(padding.c_str(), padding_size); + offset_ += padding_size; + return true; + } + + friend class ArchiveGuard; + std::ofstream ofs_; + size_t offset_; + Guard* guard_; +}; + + +class MmapInputArchive { +public: + class MmapClosure { + public: + MmapClosure(bool init = false, size_t len = 0, void* a = NULL): + initialized(init), length(len), addr(a) {}; + + ~MmapClosure() { + if (initialized && addr != MAP_FAILED) { + munmap(addr, length); + } + } + bool initialized; + size_t length; + void* addr; + }; + + MmapInputArchive(const std::string& file_path): + initialized_(false), closure_(nullptr) { + int fd = open(file_path.c_str(), O_RDONLY); + if (fd == -1) { + std::cerr << "Failed to open file " << file_path << std::endl; + return; + } + + struct stat st; + if (fstat(fd, &st) == -1) { + std::cerr << "Failed to stat file " << file_path << std::endl; + close(fd); + return; + } + + file_size_ = st.st_size; + + addr_ = mmap(NULL, file_size_, PROT_READ|PROT_WRITE, + MAP_PRIVATE, fd, 0); + if (addr_ == MAP_FAILED) { + std::cerr << "Failed to mmap file " << file_path << std::endl; + close(fd); + return; + } + + close(fd); + initialized_ = true; + offset_ = 0; + closure_ = std::make_shared(initialized_, file_size_, addr_); + }; + + ~MmapInputArchive() { + } + + bool initialized() const { + return initialized_; + } + + template + bool load(T* t, size_t align_size = 1) { + assert(offset_ + sizeof(T) <= file_size_); + char* p = (char*)addr_ + offset_; + offset_ += sizeof(T); + *t = *(reinterpret_cast(p)); + return align(align_size); + } + + char* load(size_t n, size_t align_size = 1) { + assert(offset_ + n <= file_size_); + char* p = (char*)addr_ + offset_; + offset_ += n; + align(align_size); + return p; + } + + std::shared_ptr closure() { + return closure_; + } +private: + // padding for align + bool align(size_t align) { + size_t padding_size = (offset_ + align - 1) / align * align - offset_; + if (padding_size == 0) { + return true; + } + offset_ += padding_size; + return true; + } + + bool initialized_; + size_t file_size_; + void* addr_; + size_t offset_; + std::shared_ptr closure_; +}; + +template +class MmapAllocator: public std::allocator { +public: + using value_type = T; + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using MmapClosure = typename MmapInputArchive::MmapClosure; + + MmapAllocator(): closure(nullptr) { + } + + ~MmapAllocator() { + // will call ~MmapClosure(); + } + + MmapAllocator(const MmapAllocator& m) { + this->closure = m.closure; + }; + + template + MmapAllocator(const MmapAllocator& m) { + this->closure = m.closure; + }; + + + inline pointer allocate(size_type n, const void * = 0) { + auto ret = std::allocator::allocate(n); + return ret; + } + + inline void deallocate(pointer p, size_type n) { + // mmaped memory, do not free here + if (closure + && (char*)closure->addr <= (char*)p + && (char*)p < (char*)closure->addr + closure->length) { + return; + } else { + std::allocator::deallocate(p, n); + } + } + template + struct rebind { + typedef MmapAllocator other; + }; + + void set_closure(std::shared_ptr c) { + if (closure == nullptr) { + closure = c; + } + } +public: + std::shared_ptr closure; +}; +#endif // end if __linux__ + +} // namespace phmap + +#endif // phmap_dump_h_guard_ \ No newline at end of file diff --git a/parallel_hashmap/phmap_utils.h b/parallel_hashmap/phmap_utils.h index f9c34f7..ccc078c 100644 --- a/parallel_hashmap/phmap_utils.h +++ b/parallel_hashmap/phmap_utils.h @@ -23,11 +23,8 @@ #include #include -#include -#include -#include #include "phmap_bits.h" -#include "phmap_base.h" + namespace phmap { @@ -309,107 +306,6 @@ H HashStateBase::combine(H seed, const T& v, const Ts&... vs) using HashState = HashStateBase; - -// ----------------------------------------------------------------------------- - -#define CHECK_FILE(f) { \ - if (!f.is_open()) { \ - std::cerr << "File is not open!" << std::endl; \ - return false; \ - } \ -} - -template -class ArchiveGuard { -public: - ArchiveGuard(Archive* ar): ar_(ar) { - if (ar_->guard_ == NULL) { - ar_->guard_ = this; - } - }; - ~ArchiveGuard() { - if (ar_ && ar_->guard_ == this) { - ar_->finish(); - } - } -private: - Archive* ar_; -}; - -class BinaryOutputArchive { -public: - using Guard = ArchiveGuard; - - BinaryOutputArchive(const std::string& file_path): guard_(NULL) { - ofs_.open(file_path.c_str(), std::ios_base::binary); - } - - virtual ~BinaryOutputArchive() { - finish(); - } - - bool dump(char* p, size_t sz) { - CHECK_FILE(ofs_); - ofs_.write(p, sz); - return true; - } - - template - typename std::enable_if::value, bool>::type - dump(const V& v) { - CHECK_FILE(ofs_); - ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); - return true; - } - - void finish() { - if (ofs_.is_open()) { - ofs_.close(); - } - } -private: - friend class ArchiveGuard; - std::ofstream ofs_; - Guard* guard_; -}; - - -class BinaryInputArchive { -public: - using Guard = ArchiveGuard; - - BinaryInputArchive(const std::string& file_path): guard_(NULL) { - ifs_.open(file_path.c_str(), std::ios_base::binary); - } - - virtual ~BinaryInputArchive() { - finish(); - } - - bool load(char* p, size_t sz) { - CHECK_FILE(ifs_); - ifs_.read(p, sz); - return true; - } - - template - typename std::enable_if::value, bool>::type - load(V* v) { - CHECK_FILE(ifs_); - ifs_.read(reinterpret_cast(v), sizeof(V)); - return true; - } - - void finish() { - if (ifs_.is_open()) { - ifs_.close(); - } - } -private: - friend class ArchiveGuard; - std::ifstream ifs_; - Guard* guard_; -}; } // namespace phmap diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc index e557771..da15435 100644 --- a/tests/dump_load_test.cc +++ b/tests/dump_load_test.cc @@ -1,9 +1,8 @@ #include -#include "gmock/gmock.h" #include "gtest/gtest.h" -#include "parallel_hashmap/phmap.h" +#include "parallel_hashmap/phmap_dump.h" namespace phmap { namespace container_internal { @@ -14,6 +13,9 @@ using ::phmap::flat_hash_map; using ::phmap::parallel_flat_hash_map; using ::phmap::BinaryOutputArchive; using ::phmap::BinaryInputArchive; +using ::phmap::MmapOutputArchive; +using ::phmap::MmapInputArchive; +using ::phmap::MmapAllocator; TEST(DumpLoad, FlatHashSet_uin32) { flat_hash_set st1; @@ -78,6 +80,68 @@ TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { EXPECT_EQ(1299, mp2[2130]); } +#if defined(__linux__) + +TEST(MmapDumpLoad, FlatHashMap_uint64_uint32) { + using MapType = flat_hash_map, + phmap::container_internal::hash_default_eq, + phmap::MmapAllocator< + phmap::container_internal::Pair>>; + + MapType mp1; + MmapOutputArchive ar_out("./dump.data"); + + mp1[78731] = 99; + mp1[13141] = 299; + mp1[2651] = 101; + + EXPECT_TRUE(mp1.mmap_dump(ar_out)); + MapType mp2; + MmapInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(mp2.mmap_load(ar_in)); + + EXPECT_EQ(3, mp2.size()); + EXPECT_TRUE(mp2.count(78731)); + EXPECT_TRUE(mp2.count(13141)); + EXPECT_EQ(99, mp2.at(78731)); + EXPECT_EQ(101, mp2.at(2651)); +} + +TEST(MmapDumpLoad, ParallelFlatHashMap_uint64_uint32) { + using MapType = parallel_flat_hash_map, + phmap::container_internal::hash_default_eq, + phmap::MmapAllocator< + phmap::container_internal::Pair>, + 4, + phmap::NullMutex>; + + MapType mp1; + MmapOutputArchive ar_out("./dump.data"); + + mp1[100] = 99; + mp1[300] = 299; + mp1[101] = 992; + mp1[1300] = 2991; + mp1[1130] = 299; + mp1[2130] = 1299; + + EXPECT_TRUE(mp1.mmap_dump(ar_out)); + MapType mp2; + MmapInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(mp2.mmap_load(ar_in)); + + EXPECT_EQ(6, mp2.size()); + EXPECT_EQ(99, mp2[100]); + EXPECT_EQ(299, mp2[300]); + EXPECT_EQ(299, mp2[1130]); + EXPECT_EQ(1299, mp2[2130]); +} +#endif + } } }