diff --git a/examples/dump_load.cc b/examples/dump_load.cc index efe3135..81fff62 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -49,84 +49,10 @@ void dump_load_parallel_flat_hash_map() { std::cout << "key: " << n.first << ", value: " << n.second << "\n"; } -#if defined(__linux__) -void mmap_load_uint64_uint32() { - using MapType = flat_hash_map, - phmap::container_internal::hash_default_eq, - phmap::MmapAllocator< - phmap::container_internal::Pair>>; - MapType mp1; - mp1.reserve(100); - phmap::MmapOutputArchive ar_out("./dump.data"); - // Add a new entry - mp1[100] = 99; - mp1[300] = 299; - - // Iterate and print keys and values - for (const auto& n : mp1) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - mp1.mmap_dump(ar_out); - MapType mp2; - - phmap::MmapInputArchive ar_in("./dump.data"); - mp2.mmap_load(ar_in); - mp2[849242] = 141; - mp2[11] = 1111; - // Iterate and print keys and values g|++ - for (const auto& n : mp2) - std::cout << n.first << "'s value is: " << n.second << "\n"; -} - -void mmap_load_parallel_flat_hash_map() { - using MapType = parallel_flat_hash_map, - phmap::container_internal::hash_default_eq, - phmap::MmapAllocator< - phmap::container_internal::Pair>, - 4, - phmap::NullMutex>; - - MapType mp1; - phmap::MmapOutputArchive ar_out("./dump.data"); - - // Add a new entry - mp1[100] = 99; - mp1[300] = 299; - mp1[101] = 992; - mp1[1300] = 2991; - mp1[1130] = 299; - mp1[2130] = 1299; - // Iterate and print - for (const auto& n : mp1) - std::cout << "key: " << n.first << ", value: " << n.second << "\n"; - - mp1.mmap_dump(ar_out); - MapType mp2; - phmap::MmapInputArchive ar_in("./dump.data"); - - mp2.mmap_load(ar_in); - std::cout << "[debug] map capacity: " << mp2.capacity() << ", size: " << mp2.size() << std::endl; - for (size_t i = 0; i < 100; i ++) { - mp2[6771 + i] = i; - } - std::cout << "[debug] map capacity: " << mp2.capacity() << ", size: " << mp2.size() << std::endl; - - for (const auto& n : mp2) - std::cout << "key: " << n.first << ", value: " << n.second << "\n"; -} -#endif - int main() { dump_load_uint64_uint32(); dump_load_parallel_flat_hash_map(); - -#if defined(__linux__) - mmap_load_uint64_uint32(); - mmap_load_parallel_flat_hash_map(); -#endif return 0; } diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 404ac22..2a2bcc2 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1540,12 +1540,6 @@ public: template bool load(InputArchive&); - template - bool mmap_dump(OutputArchive&); - - template - bool mmap_load(MmapInputArchive&); - void rehash(size_t n) { if (n == 0 && capacity_ == 0) return; if (n == 0 && size_ == 0) { @@ -3157,11 +3151,6 @@ public: template bool load(InputArchive& ar); - template - bool mmap_dump(OutputArchive& ar); - - template - bool mmap_load(InputArchive& ar); private: template friend struct phmap::container_internal::hashtable_debug_internal::HashtableDebugAccess; diff --git a/parallel_hashmap/phmap_dump.h b/parallel_hashmap/phmap_dump.h index b75262a..bd36ac9 100644 --- a/parallel_hashmap/phmap_dump.h +++ b/parallel_hashmap/phmap_dump.h @@ -111,79 +111,6 @@ bool raw_hash_set::load(InputArchive& ar) { return true; } - -template -template -bool raw_hash_set::mmap_dump(OutputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable::value, - "value_type should be dumpable"); - - typename OutputArchive::Guard guard(&ar); - size_t align_size = Layout::Alignment(); - if (!ar.dump(size_, align_size)) { - std::cerr << "Failed to dump size_" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.dump(capacity_, align_size)) { - std::cerr << "Failed to dump capacity_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1), align_size)) { - - std::cerr << "Failed to dump ctrl_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast(slots_), - sizeof(slot_type) * capacity_, align_size)) { - std::cerr << "Failed to dump slot_" << std::endl; - return false; - } - return true; -} - -template -template -bool raw_hash_set::mmap_load(MmapInputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable::value, - "value_type should be dumpable"); - - assert(ar.initialized()); - auto closure = ar.closure(); - this->alloc_ref().set_closure(closure); - size_t align_size = Layout::Alignment(); - if (!ar.load(&size_, align_size)) { - std::cerr << "Failed to load size!" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.load(&capacity_, align_size)) { - std::cerr << "Failed to load capacity!" << std::endl; - return false; - } - - - if (std::is_same>::value) { - infoz_ = Sample(); - } - reset_growth_left(); - infoz_.RecordStorageChanged(size_, capacity_); - - char* p_ctrl = ar.load( - sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1), align_size); - ctrl_ = reinterpret_cast(p_ctrl); - - char* p_slots = ar.load(sizeof(slot_type) * capacity_, align_size); - slots_ = reinterpret_cast(p_slots); - return true; -} - - ////// parallel_hash_set template class RefSet, @@ -241,67 +168,6 @@ bool parallel_hash_set::load(InputArch } return true; } - -template class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -template -bool parallel_hash_set::mmap_dump(OutputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable::value, - "value_type should be dumpable"); - - typename OutputArchive::Guard guard(&ar); - size_t align_size = EmbeddedSet::Layout::Alignment(); - if (! ar.dump(subcnt(), align_size)) { - std::cerr << "Failed to dump meta!" << std::endl; - return false; - } - for (size_t i = 0; i < sets_.size(); ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.mmap_dump(ar)) { - std::cerr << "Failed to dump submap " << i << std::endl; - return false; - } - } - return true; -} - -template class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -template -bool parallel_hash_set::mmap_load(InputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable::value, - "value_type should be dumpable"); - - assert(ar.initialized()); - auto closure = ar.closure(); - this->alloc_ref().set_closure(closure); - size_t submap_count = 0; - size_t align_size = EmbeddedSet::Layout::Alignment(); - if (!ar.load(&submap_count, align_size)) { - std::cerr << "Failed to load submap count!" << std::endl; - return false; - } - - if (submap_count != subcnt()) { - std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; - return false; - } - - for (size_t i = 0; i < submap_count; ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.mmap_load(ar)) { - std::cerr << "Failed to load submap " << i << std::endl; - return false; - } - } - return true; -} } // namesapce container_internal @@ -417,219 +283,6 @@ private: Guard* guard_; }; -#if defined(__linux__) // only support linux's mmap now -// mmap dump && mmap load -#include -#include -#include -#include - -class MmapOutputArchive { -public: - using Guard = ArchiveGuard; - - MmapOutputArchive(const std::string& file_path): offset_(0), guard_(NULL) { - ofs_.open(file_path.c_str(), std::ios_base::binary); - } - - virtual ~MmapOutputArchive() { - finish(); - } - - bool dump(char* p, size_t sz, size_t align_size) { - CHECK_FILE(ofs_); - ofs_.write(p, sz); - offset_ += sz; - return align(align_size); - } - - template - typename std::enable_if::value, bool>::type - dump(const V& v, size_t align_size) { - CHECK_FILE(ofs_); - ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); - offset_ += sizeof(V); - return align(align_size); - } - - void finish() { - if (ofs_.is_open()) { - ofs_.close(); - offset_ = 0; - } - } -private: - // padding for align - bool align(size_t align) { - size_t padding_size = (offset_ + align - 1) / align * align - offset_; - if (padding_size == 0) { - return true; - } - std::string padding(padding_size, '\0'); - ofs_.write(padding.c_str(), padding_size); - offset_ += padding_size; - return true; - } - - friend class ArchiveGuard; - std::ofstream ofs_; - size_t offset_; - Guard* guard_; -}; - - -class MmapInputArchive { -public: - class MmapClosure { - public: - MmapClosure(bool init = false, size_t len = 0, void* a = NULL): - initialized(init), length(len), addr(a) {}; - - ~MmapClosure() { - if (initialized && addr != MAP_FAILED) { - munmap(addr, length); - } - } - bool initialized; - size_t length; - void* addr; - }; - - MmapInputArchive(const std::string& file_path): - initialized_(false), closure_(nullptr) { - int fd = open(file_path.c_str(), O_RDONLY); - if (fd == -1) { - std::cerr << "Failed to open file " << file_path << std::endl; - return; - } - - struct stat st; - if (fstat(fd, &st) == -1) { - std::cerr << "Failed to stat file " << file_path << std::endl; - close(fd); - return; - } - - file_size_ = st.st_size; - - addr_ = mmap(NULL, file_size_, PROT_READ|PROT_WRITE, - MAP_PRIVATE, fd, 0); - if (addr_ == MAP_FAILED) { - std::cerr << "Failed to mmap file " << file_path << std::endl; - close(fd); - return; - } - - close(fd); - initialized_ = true; - offset_ = 0; - closure_ = std::make_shared(initialized_, file_size_, addr_); - }; - - ~MmapInputArchive() { - } - - bool initialized() const { - return initialized_; - } - - template - bool load(T* t, size_t align_size = 1) { - assert(offset_ + sizeof(T) <= file_size_); - char* p = (char*)addr_ + offset_; - offset_ += sizeof(T); - *t = *(reinterpret_cast(p)); - return align(align_size); - } - - char* load(size_t n, size_t align_size = 1) { - assert(offset_ + n <= file_size_); - char* p = (char*)addr_ + offset_; - offset_ += n; - align(align_size); - return p; - } - - std::shared_ptr closure() { - return closure_; - } -private: - // padding for align - bool align(size_t align) { - size_t padding_size = (offset_ + align - 1) / align * align - offset_; - if (padding_size == 0) { - return true; - } - offset_ += padding_size; - return true; - } - - bool initialized_; - size_t file_size_; - void* addr_; - size_t offset_; - std::shared_ptr closure_; -}; - -template -class MmapAllocator: public std::allocator { -public: - using value_type = T; - using pointer = T*; - using const_pointer = const T*; - using reference = T&; - using const_reference = const T&; - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - using MmapClosure = typename MmapInputArchive::MmapClosure; - - MmapAllocator(): closure(nullptr) { - } - - ~MmapAllocator() { - // will call ~MmapClosure(); - } - - MmapAllocator(const MmapAllocator& m) { - this->closure = m.closure; - }; - - template - MmapAllocator(const MmapAllocator& m) { - this->closure = m.closure; - }; - - - inline pointer allocate(size_type n, const void * = 0) { - auto ret = std::allocator::allocate(n); - return ret; - } - - inline void deallocate(pointer p, size_type n) { - // mmaped memory, do not free here - if (closure - && (char*)closure->addr <= (char*)p - && (char*)p < (char*)closure->addr + closure->length) { - return; - } else { - std::allocator::deallocate(p, n); - } - } - template - struct rebind { - typedef MmapAllocator other; - }; - - void set_closure(std::shared_ptr c) { - if (closure == nullptr) { - closure = c; - } - } -public: - std::shared_ptr closure; -}; -#endif // end if __linux__ - } // namespace phmap -#endif // phmap_dump_h_guard_ \ No newline at end of file +#endif // phmap_dump_h_guard_ diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc index d3f9b5f..4933406 100644 --- a/tests/dump_load_test.cc +++ b/tests/dump_load_test.cc @@ -13,9 +13,6 @@ using ::phmap::flat_hash_map; using ::phmap::parallel_flat_hash_map; using ::phmap::BinaryOutputArchive; using ::phmap::BinaryInputArchive; -using ::phmap::MmapOutputArchive; -using ::phmap::MmapInputArchive; -using ::phmap::MmapAllocator; TEST(DumpLoad, FlatHashSet_uin32) { flat_hash_set st1; @@ -79,67 +76,6 @@ TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { EXPECT_EQ(1299, mp2[2130]); } -#if defined(__linux__) - -TEST(MmapDumpLoad, FlatHashMap_uint64_uint32) { - using MapType = flat_hash_map, - phmap::container_internal::hash_default_eq, - phmap::MmapAllocator< - phmap::container_internal::Pair>>; - - MapType mp1; - MmapOutputArchive ar_out("./dump.data"); - - mp1[78731] = 99; - mp1[13141] = 299; - mp1[2651] = 101; - - EXPECT_TRUE(mp1.mmap_dump(ar_out)); - MapType mp2; - MmapInputArchive ar_in("./dump.data"); - - EXPECT_TRUE(mp2.mmap_load(ar_in)); - - EXPECT_EQ(3, mp2.size()); - EXPECT_TRUE(mp2.count(78731)); - EXPECT_TRUE(mp2.count(13141)); - EXPECT_EQ(99, mp2.at(78731)); - EXPECT_EQ(101, mp2.at(2651)); -} - -TEST(MmapDumpLoad, ParallelFlatHashMap_uint64_uint32) { - using MapType = parallel_flat_hash_map, - phmap::container_internal::hash_default_eq, - phmap::MmapAllocator< - phmap::container_internal::Pair>, - 4, - phmap::NullMutex>; - - MapType mp1; - MmapOutputArchive ar_out("./dump.data"); - - mp1[100] = 99; - mp1[300] = 299; - mp1[101] = 992; - mp1[1300] = 2991; - mp1[1130] = 299; - mp1[2130] = 1299; - - EXPECT_TRUE(mp1.mmap_dump(ar_out)); - MapType mp2; - MmapInputArchive ar_in("./dump.data"); - - EXPECT_TRUE(mp2.mmap_load(ar_in)); - EXPECT_EQ(6, mp2.size()); - EXPECT_EQ(99, mp2[100]); - EXPECT_EQ(299, mp2[300]); - EXPECT_EQ(299, mp2[1130]); - EXPECT_EQ(1299, mp2[2130]); -} -#endif - } } }