From 61f1bdb037b336ebf6225fec1d4b193031a766c9 Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Wed, 21 Aug 2019 09:01:56 +0800 Subject: [PATCH] use single file to store data for parallel hash --- examples/dump_load.cc | 8 ++-- parallel_hashmap/phmap.h | 38 +++++++++--------- parallel_hashmap/phmap_utils.h | 70 ++++++---------------------------- tests/dump_load_test.cc | 10 ++--- 4 files changed, 41 insertions(+), 85 deletions(-) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index 4c92113..6369c0c 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -27,7 +27,7 @@ void dump_load_uint64_uint32() { void dump_load_parallel_flat_hash_map() { parallel_flat_hash_map mp1; - phmap::OutputArchiveWrapper w_out("./"); + phmap::BinaryOutputArchive ar_out("./dump.data"); // Add a new entry mp1[100] = 99; @@ -40,11 +40,11 @@ void dump_load_parallel_flat_hash_map() { for (const auto& n : mp1) std::cout << "key: " << n.first << ", value: " << n.second << "\n"; - mp1.dump(w_out); + mp1.dump(ar_out); parallel_flat_hash_map mp2; - phmap::InputArchiveWrapper w_in("./"); + phmap::BinaryInputArchive ar_in("./dump.data"); - mp2.load(w_in); + mp2.load(ar_in); for (const auto& n : mp2) std::cout << "key: " << n.first << ", value: " << n.second << "\n"; } diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 05e8b25..69c3baf 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1553,7 +1553,7 @@ public: return false; } if (!ar.dump(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { std::cerr << "Failed to dump ctrl_" << std::endl; return false; @@ -1583,7 +1583,7 @@ public: // allocate memory for ctrl_ and slots_ initialize_slots(); if (!ar.load(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { std::cerr << "Failed to load ctrl" << std::endl; return false; } @@ -3199,40 +3199,44 @@ public: a.swap(b); } - template + template typename std::enable_if::value, bool>::type - dump(OutputArchiveWrapper& w) { + dump(OutputArchive& ar) { + typename OutputArchive::Guard guard(&ar); + if (! ar.dump(subcnt())) { + std::cerr << "Failed to dump meta!" << std::endl; + return false; + } for (size_t i = 0; i < sets_.size(); ++i) { auto& inner = sets_[i]; - auto ar = w.create_archive(i); typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.dump(*ar)) { + if (!inner.set_.dump(ar)) { std::cerr << "Failed to dump submap " << i << std::endl; return false; } } - - if (! w.dump_meta(subcnt())) { - std::cerr << "Failed to dump meta!" << std::endl; - return false; - } return true; } - template + template typename std::enable_if::value, bool>::type - load(InputArchiveWrapper& w) { - size_t submap_count = w.load_meta(); + load(InputArchive& ar) { + typename InputArchive::Guard guard(&ar); + size_t submap_count = 0; + if (!ar.load(&submap_count)) { + std::cerr << "Failed to load submap count!" << std::endl; + return false; + } if (submap_count != subcnt()) { std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; return false; } - for (size_t i = 0; i < sets_.size(); ++i) { - auto ar = w.create_archive(i); + for (size_t i = 0; i < submap_count; ++i) { auto& inner = sets_[i]; - if (!inner.set_.load(*ar)) { + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.load(ar)) { std::cerr << "Failed to load submap " << i << std::endl; return false; } diff --git a/parallel_hashmap/phmap_utils.h b/parallel_hashmap/phmap_utils.h index 9146057..f9c34f7 100644 --- a/parallel_hashmap/phmap_utils.h +++ b/parallel_hashmap/phmap_utils.h @@ -322,9 +322,13 @@ using HashState = HashStateBase; template class ArchiveGuard { public: - ArchiveGuard(Archive* ar): ar_(ar) {}; + ArchiveGuard(Archive* ar): ar_(ar) { + if (ar_->guard_ == NULL) { + ar_->guard_ = this; + } + }; ~ArchiveGuard() { - if (ar_) { + if (ar_ && ar_->guard_ == this) { ar_->finish(); } } @@ -336,7 +340,7 @@ class BinaryOutputArchive { public: using Guard = ArchiveGuard; - BinaryOutputArchive(const std::string& file_path) { + BinaryOutputArchive(const std::string& file_path): guard_(NULL) { ofs_.open(file_path.c_str(), std::ios_base::binary); } @@ -364,7 +368,9 @@ public: } } private: + friend class ArchiveGuard; std::ofstream ofs_; + Guard* guard_; }; @@ -372,7 +378,7 @@ class BinaryInputArchive { public: using Guard = ArchiveGuard; - BinaryInputArchive(const std::string& file_path) { + BinaryInputArchive(const std::string& file_path): guard_(NULL) { ifs_.open(file_path.c_str(), std::ios_base::binary); } @@ -400,62 +406,10 @@ public: } } private: + friend class ArchiveGuard; std::ifstream ifs_; + Guard* guard_; }; - -template -class OutputArchiveWrapper { -public: - using SubArchive = T; - OutputArchiveWrapper(const std::string& dir): dir_(dir) { - } - - virtual ~OutputArchiveWrapper() { - - } - - bool dump_meta(size_t subcnt) { - auto ar = std::make_shared(dir_ + "/meta.dump"); - typename SubArchive::Guard guard(ar.get()); - ar->dump(subcnt); - return true; - } - - std::shared_ptr create_archive(size_t i) { - std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump"; - return std::make_shared(file_path); - } -private: - std::string dir_; -}; - -template -class InputArchiveWrapper { -public: - using SubArchive = T; - InputArchiveWrapper(const std::string& dir): dir_(dir) { - } - - virtual ~InputArchiveWrapper() { - - } - - size_t load_meta() { - size_t subcnt = 0; - auto ar = std::make_shared(dir_ + "/meta.dump"); - typename SubArchive::Guard guard(ar.get()); - ar->load(&subcnt); - return subcnt; - } - - std::shared_ptr create_archive(size_t i) { - std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump"; - return std::make_shared(file_path); - } -private: - std::string dir_; -}; - } // namespace phmap diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc index e162ba0..e557771 100644 --- a/tests/dump_load_test.cc +++ b/tests/dump_load_test.cc @@ -14,8 +14,6 @@ using ::phmap::flat_hash_map; using ::phmap::parallel_flat_hash_map; using ::phmap::BinaryOutputArchive; using ::phmap::BinaryInputArchive; -using ::phmap::OutputArchiveWrapper; -using ::phmap::InputArchiveWrapper; TEST(DumpLoad, FlatHashSet_uin32) { flat_hash_set st1; @@ -58,7 +56,7 @@ TEST(DumpLoad, FlatHashMap_uint64_uint32) { TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { parallel_flat_hash_map mp1; - OutputArchiveWrapper w_out("./"); + BinaryOutputArchive ar_out("./dump.data"); mp1[100] = 99; mp1[300] = 299; @@ -67,11 +65,11 @@ TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { mp1[1130] = 299; mp1[2130] = 1299; - EXPECT_TRUE(mp1.dump(w_out)); + EXPECT_TRUE(mp1.dump(ar_out)); parallel_flat_hash_map mp2; - InputArchiveWrapper w_in("./"); + BinaryInputArchive ar_in("./dump.data"); - EXPECT_TRUE(mp2.load(w_in)); + EXPECT_TRUE(mp2.load(ar_in)); EXPECT_EQ(6, mp2.size()); EXPECT_EQ(99, mp2[100]);