From dd9af7f8a5049f5be39b1d202c5e4068a76b5462 Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Fri, 16 Aug 2019 21:24:02 +0800 Subject: [PATCH] refine code --- examples/dump_load.cc | 53 +++++--- parallel_hashmap/phmap.h | 237 ++++++++++++--------------------- parallel_hashmap/phmap_base.h | 50 ------- parallel_hashmap/phmap_utils.h | 191 +++++++++++++++++++++++++- 4 files changed, 310 insertions(+), 221 deletions(-) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index 8f652ff..8c20d52 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -8,7 +8,7 @@ using phmap::parallel_flat_hash_map; void dump_load_string_string() { flat_hash_map mp1; - + phmap::BinaryOutputArchive ar_out("./dump.data"); // Add a new entry mp1["key-1"] = "value-1"; mp1["key-2"] = "value-2"; @@ -17,10 +17,12 @@ void dump_load_string_string() { for (const auto& n : mp1) std::cout << n.first << "'s value is: " << n.second << "\n"; - mp1.dump("./dump.data"); + mp1.dump(ar_out); flat_hash_map mp2; - mp2.load("./dump.data"); + phmap::BinaryInputArchive ar_in("./dump.data"); + + mp2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; @@ -30,7 +32,7 @@ void dump_load_string_string() { void dump_load_uint64_uint32() { flat_hash_map mp1; - + phmap::BinaryOutputArchive ar_out("./dump.data"); // Add a new entry mp1[100] = 99; mp1[300] = 299; @@ -39,10 +41,10 @@ void dump_load_uint64_uint32() { for (const auto& n : mp1) std::cout << n.first << "'s value is: " << n.second << "\n"; - mp1.dump("./dump.data"); + mp1.dump(ar_out); flat_hash_map mp2; - - mp2.load("./dump.data"); + phmap::BinaryInputArchive ar_in("./dump.data"); + mp2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; @@ -52,7 +54,7 @@ void dump_load_uint64_uint32() { void dump_load_string_uint32() { flat_hash_map mp1; - + phmap::BinaryOutputArchive ar_out("./dump.data"); // Add a new entry mp1["key-1"] = 99; mp1["key-2"] = 299; @@ -61,10 +63,11 @@ void dump_load_string_uint32() { for (const auto& n : mp1) std::cout << n.first << "'s value is: " << n.second << "\n"; - mp1.dump("./dump.data"); + mp1.dump(ar_out); flat_hash_map mp2; + phmap::BinaryInputArchive ar_in("./dump.data"); - mp2.load("./dump.data"); + mp2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; @@ -74,6 +77,7 @@ void dump_load_string_uint32() { void dump_load_uint32_string() { flat_hash_map mp1; + phmap::BinaryOutputArchive ar_out("./dump.data"); // Add a new entry mp1[100] = "hello"; @@ -83,10 +87,11 @@ void dump_load_uint32_string() { for (const auto& n : mp1) std::cout << n.first << "'s value is: " << n.second << "\n"; - mp1.dump("./dump.data"); + mp1.dump(ar_out); flat_hash_map mp2; + phmap::BinaryInputArchive ar_in("./dump.data"); - mp2.load("./dump.data"); + mp2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; @@ -96,7 +101,8 @@ void dump_load_uint32_string() { void dump_load_string() { flat_hash_set st1; - + phmap::BinaryOutputArchive ar_out("./dump.data"); + // Add a new entry st1.insert("hello"); st1.insert("world"); @@ -105,10 +111,11 @@ void dump_load_string() { for (const auto& n : st1) std::cout << "value: " << n << "\n"; - st1.dump("./dump.data"); + st1.dump(ar_out); flat_hash_set st2; + phmap::BinaryInputArchive ar_in("./dump.data"); - st2.load("./dump.data"); + st2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : st2) std::cout << "value: " << n << "\n"; @@ -118,7 +125,8 @@ void dump_load_string() { void dump_load_uint64() { flat_hash_set st1; - + phmap::BinaryOutputArchive ar_out("./dump.data"); + // Add a new entry st1.insert(878); st1.insert(1424); @@ -127,10 +135,11 @@ void dump_load_uint64() { for (const auto& n : st1) std::cout << "value: " << n << "\n"; - st1.dump("./dump.data"); + st1.dump(ar_out); flat_hash_set st2; + phmap::BinaryInputArchive ar_in("./dump.data"); - st2.load("./dump.data"); + st2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : st2) std::cout << "value: " << n << "\n"; @@ -140,7 +149,8 @@ void dump_load_uint64() { void dump_load_parallel_flat_hash_map() { parallel_flat_hash_map mp1; - + phmap::OutputArchiveWrapper w_out("./"); + // Add a new entry mp1[100] = 99; mp1[300] = 299; @@ -152,10 +162,11 @@ void dump_load_parallel_flat_hash_map() { for (const auto& n : mp1) std::cout << "key: " << n.first << ", value: " << n.second << "\n"; - mp1.dump("./dump"); + mp1.dump(w_out); parallel_flat_hash_map mp2; + phmap::InputArchiveWrapper w_in("./"); - mp2.load("./dump"); + mp2.load(w_in); for (const auto& n : mp2) std::cout << "key: " << n.first << ", value: " << n.second << "\n"; } diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 34e7552..175a758 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1537,148 +1537,111 @@ public: } } - template + template typename std::enable_if::value, bool>::type - dump(const std::string& dump_file) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - if (size_ == 0) { - std::cout << "Empty set, nothing to dump" << std::endl; - return true; - } - assert(slots_ != nullptr); - std::ofstream ofs(dump_file); - if (!ofs.is_open()) { - std::cout << "Failed to open dump file " << dump_file << std::endl; + dump(OutputArchive& ar) { + typename OutputArchive::Guard guard(&ar); + if (!ar.dump(size_)) { + std::cerr << "Failed to dump size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.dump(capacity_)) { + std::cerr << "Failed to dump capacity_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(ctrl_), sizeof(ctrl_t) * capacity_)) { + std::cerr << "Failed to dump ctrl_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(slots_), sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to dump slot_" << std::endl; return false; } - ofs.write(reinterpret_cast(&size_), sizeof(size_)); - ofs.write(reinterpret_cast(&capacity_), sizeof(capacity_)); - ofs.write(reinterpret_cast(ctrl_), capacity_ * sizeof(ctrl_t)); - ofs.write(reinterpret_cast(slots_), capacity_ * sizeof(slot_type)); - ofs.close(); return true; } - template + template typename std::enable_if::value, bool>::type - load(const std::string& load_file) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - std::ifstream ifs(load_file); - if (!ifs.is_open()) { - std::cerr << "Failed to open load file " << load_file << std::endl; + load(InputArchive& ar) { + typename InputArchive::Guard guard(&ar); + if (!ar.load(&size_)){ + std::cerr << "Failed to load size_" << std::endl; return false; } - // get file size - ifs.seekg(0, std::ios::end); - size_t file_size = ifs.tellg(); - ifs.seekg(0, std::ios::beg); - if (file_size <= sizeof(size_) + sizeof(capacity_)) { - std::cerr << "Invalid file format. file size: " << file_size << ", size_: " - << size_ << ", capacity_: " << capacity_ << ", slot type size: " - << sizeof(slot_type); - return false; + if (size_ == 0) { + return true; } - - ifs.read(reinterpret_cast(&size_), sizeof(size_)); - ifs.read(reinterpret_cast(&capacity_), sizeof(capacity_)); - if (file_size != sizeof(size_) + sizeof(capacity_) + capacity_ * sizeof(ctrl_t) - + capacity_ * sizeof(slot_type)) { - std::cerr << "Invalid file format. file size: " << file_size << ", size_: " - << size_ << ", capacity_: " << capacity_ << ", slot type size: " - << sizeof(slot_type); + if (!ar.load(&capacity_)) { + std::cerr << "Failed to load capacity_" << std::endl; return false; } // allocate memory for ctrl_ and slots_ initialize_slots(); - - ifs.read(reinterpret_cast(ctrl_), capacity_ * sizeof(ctrl_t)); - ifs.read(reinterpret_cast(slots_), capacity_ * sizeof(slot_type)); - ifs.close(); + if (!ar.load(reinterpret_cast(ctrl_), sizeof(ctrl_t) * capacity_)) { + std::cerr << "Failed to load ctrl" << std::endl; + return false; + } + if (!ar.load(reinterpret_cast(slots_), sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to load slot" << std::endl; + return false; + } return true; } // V will be V for hash_set and std::pair for hash_map - template + template typename std::enable_if::value && type_traits_internal::IsStringOrArithmeticType::value, bool>::type - dump(const std::string& dump_file) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - if (size_ == 0) { - std::cout << "Empty set, nothing to dump" << std::endl; + dump(OutputArchive& ar) { + typename OutputArchive::Guard guard(&ar); + if (!ar.template dump(size_)) { + std::cerr << "Failed to dump size" << std::endl; + return false; + } + if (size_ == 0) { return true; } - assert(slots_ != nullptr); - std::ofstream ofs(dump_file); - if (!ofs.is_open()) { - std::cout << "Failed to open dump file " << dump_file << std::endl; - return false; - } - - ofs.write(reinterpret_cast(&size_), sizeof(size_)); - for (auto it = this->begin(); it != this->end(); ++it) { - type_traits_internal::Archive::dump(*it, &ofs); - } - ofs.close(); - return true; - } - - template - typename std::enable_if::value - && type_traits_internal::IsStringOrArithmeticType::value, bool>::type - load(const std::string& load_file = "") noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - - std::ifstream ifs(load_file); - if (!ifs.is_open()) { - std::cerr << "Failed to open load file " << load_file << std::endl; - return false; - } - - size_t total_count = 0; - ifs.read((char*)&total_count, sizeof(total_count)); - - for (size_t i = 0; i < total_count; i ++) { - if (ifs.eof()) { - std::cerr << "Data is not enough, total_count: " << total_count - << ", meet eof at index: " << i << std::endl; + if (!ar.template dump(*it)) { + std::cerr << "Failed to dump element" << std::endl; return false; } - V v; - type_traits_internal::Archive::load(ifs, &v); - this->insert(v); } - ifs.close(); return true; } - template + template + typename std::enable_if::value + && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + load(InputArchive& ar) { + typename InputArchive::Guard guard(&ar); + size_t sz = 0; + ar.template load(&sz); + for (size_t i = 0; i < sz; i ++) { + V v; + if (!ar.template load(&v)) { + std::cerr << "Failed to load element " << i << std::endl; + return false; + } + this->insert(v); + } + return true; + } + + template typename std::enable_if::value, bool>::type - dump(const std::string&) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { + dump(OutputArchive&) { std::cerr << "Does not support this type now!" << std::endl; - std::abort(); return false; } - template + template typename std::enable_if::value, bool>::type - load(const std::string&) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - std::cerr << "Does not support this type now!" << std::endl; - std::abort(); + load(InputArchive&) { + std::cerr << "Does not support this type now!" << std::endl; return false; } @@ -3287,63 +3250,39 @@ public: a.swap(b); } - bool dump(const std::string& dump_dir) { - for (size_t i = 0; i < sets_.size(); ++i) { + template + bool dump(OutputArchiveWrapper& w) { + for (size_t i = 0; i < sets_.size(); ++i) { auto& inner = sets_[i]; - if (inner.set_.size() == 0) { - continue; - } - const std::string& dump_path = dump_dir + "/submap_" + std::to_string(i) + ".dump"; + auto ar = w.create_archive(i); typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.dump(dump_path)) { + if (!inner.set_.dump(*ar)) { + std::cerr << "Failed to dump submap " << i << std::endl; return false; } } - std::ofstream fout(dump_dir + "/dump.meta"); - fout << sets_.size(); // submap count - fout.close(); + + if (! w.dump_meta(subcnt())) { + std::cerr << "Failed to dump meta!" << std::endl; + return false; + } return true; } - bool load(const std::string& load_dir) { - std::ifstream fin(load_dir + "/dump.meta"); - if (! fin.is_open()) { - std::cerr << "Failed to find dump.meta in dir " << load_dir << std::endl; - return false; - } - - size_t submap_count = 0; - fin >> submap_count; - if (submap_count <= 0) { - std::cerr << "Invalid submap count: " << submap_count << std::endl; - return false; - } - - fin.close(); + template + bool load(InputArchiveWrapper& w) { + size_t submap_count = w.load_meta(); if (submap_count != subcnt()) { std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; return false; } - auto file_exists = [] (const std::string& file_name) -> bool { - std::ifstream fin(file_name); - if (fin.is_open()) { - fin.close(); - return true; - } else { - fin.close(); - return false; - } - }; - - for (size_t i = 0; i < submap_count; ++i) { + for (size_t i = 0; i < sets_.size(); ++i) { + auto ar = w.create_archive(i); auto& inner = sets_[i]; - const std::string& load_file = load_dir + "/submap_" + std::to_string(i) + ".dump"; - if (!file_exists(load_file)) { - continue; - } - if (!inner.set_.load(load_file)) { + if (!inner.set_.load(*ar)) { + std::cerr << "Failed to load submap " << i << std::endl; return false; } } diff --git a/parallel_hashmap/phmap_base.h b/parallel_hashmap/phmap_base.h index 2045d2a..8f03623 100644 --- a/parallel_hashmap/phmap_base.h +++ b/parallel_hashmap/phmap_base.h @@ -101,56 +101,6 @@ struct IsStringOrArithmeticType { || std::is_same::second_type, std::string>::value)); }; -// only support std::is_arithmetic or std::string types -template -struct Archive { - template - static typename std::enable_if::value, void>::type - dump(const V& v, std::ofstream* ofs) { - ofs->write(reinterpret_cast(const_cast(&v)), sizeof(V)); - } - - template - static typename std::enable_if::value, void>::type - load(std::ifstream& ifs, V* v) { - ifs.read(reinterpret_cast(v), sizeof(V)); - } - - template - static typename std::enable_if::type>::value, void>::type - dump(const V& v, std::ofstream* ofs) { - uint32_t sz = v.length(); - ofs->write(reinterpret_cast(&sz), sizeof(sz)); - ofs->write(const_cast(v.data()), sz); - } - - template - static typename std::enable_if::type>::value, void>::type - load(std::ifstream& ifs, V* v) { - uint32_t sz = 0; - ifs.read(reinterpret_cast(&sz), sizeof(sz)); - const_cast(v)->resize(sz); - ifs.read(const_cast(v->data()), sz); - } - - template - static typename std::enable_if::value && IsStringOrArithmeticType::value, void>::type - dump(const V& v, std::ofstream* ofs) { - dump::first_type>(v.first, ofs); - dump::second_type>(v.second, ofs); - } - - template - static typename std::enable_if::value && IsStringOrArithmeticType::value, void>::type - load(std::ifstream& ifs, V* v) { - using first_type = typename PairTrait::first_type; - using second_type = typename PairTrait::second_type; - load(ifs, const_cast(&v->first)); - load(ifs, const_cast(&v->second)); - } -}; - - template struct VoidTImpl { using type = void; diff --git a/parallel_hashmap/phmap_utils.h b/parallel_hashmap/phmap_utils.h index ccc078c..3807498 100644 --- a/parallel_hashmap/phmap_utils.h +++ b/parallel_hashmap/phmap_utils.h @@ -23,8 +23,11 @@ #include #include +#include +#include +#include #include "phmap_bits.h" - +#include "phmap_base.h" namespace phmap { @@ -306,6 +309,192 @@ H HashStateBase::combine(H seed, const T& v, const Ts&... vs) using HashState = HashStateBase; + +// ----------------------------------------------------------------------------- + +#define CHECK_FILE(f) { \ + if (!f.is_open()) { \ + std::cerr << "File is not open!" << std::endl; \ + return false; \ + } \ +} + +template +class ArchiveGuard { +public: + ArchiveGuard(Archive* ar): ar_(ar) {}; + ~ArchiveGuard() { + if (ar_) { + ar_->finish(); + } + } +private: + Archive* ar_; +}; + +class BinaryOutputArchive { +public: + using Guard = ArchiveGuard; + + BinaryOutputArchive(const std::string& file_path) { + ofs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~BinaryOutputArchive() { + finish(); + } + + bool dump(char* p, size_t sz) { + CHECK_FILE(ofs_); + ofs_.write(p, sz); + return true; + } + + template + typename std::enable_if::value, bool>::type + dump(const V& v) { + CHECK_FILE(ofs_); + ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); + return true; + } + + template + typename std::enable_if::type>::value, bool>::type + dump(const V& v) { + CHECK_FILE(ofs_); + uint32_t sz = v.length(); + ofs_.write(reinterpret_cast(&sz), sizeof(sz)); + ofs_.write(const_cast(v.data()), sz); + return true; + } + + template + typename std::enable_if::value + && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + dump(const V& v) { + return dump::first_type>(v.first) + && dump::second_type>(v.second); + } + + void finish() { + if (ofs_.is_open()) { + ofs_.close(); + } + } +private: + std::ofstream ofs_; +}; + + +class BinaryInputArchive { +public: + using Guard = ArchiveGuard; + + BinaryInputArchive(const std::string& file_path) { + ifs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~BinaryInputArchive() { + finish(); + } + + bool load(char* p, size_t sz) { + CHECK_FILE(ifs_); + ifs_.read(p, sz); + return true; + } + + template + typename std::enable_if::value, bool>::type + load(V* v) { + CHECK_FILE(ifs_); + ifs_.read(reinterpret_cast(v), sizeof(V)); + return true; + } + + template + typename std::enable_if::type>::value, bool>::type + load(V* v) { + CHECK_FILE(ifs_); + uint32_t sz = 0; + ifs_.read(reinterpret_cast(&sz), sizeof(sz)); + const_cast(v)->resize(sz); + ifs_.read(const_cast(v->data()), sz); + return true; + } + + template + typename std::enable_if::value + && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + load(V* v) { + using first_type = typename type_traits_internal::PairTrait::first_type; + using second_type = typename type_traits_internal::PairTrait::second_type; + return load(const_cast(&v->first)) + && load(const_cast(&v->second)); + } + + void finish() { + if (ifs_.is_open()) { + ifs_.close(); + } + } +private: + std::ifstream ifs_; +}; + +template +class OutputArchiveWrapper { +public: + using SubArchive = T; + OutputArchiveWrapper(const std::string& dir): dir_(dir) { + } + + virtual ~OutputArchiveWrapper() { + + } + + bool dump_meta(size_t subcnt) { + auto ar = std::make_shared(dir_ + "/meta.dump"); + typename SubArchive::Guard guard(ar.get()); + ar->dump(subcnt); + return true; + } + + std::shared_ptr create_archive(size_t i) { + std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump"; + return std::make_shared(file_path); + } +private: + std::string dir_; +}; + +template +class InputArchiveWrapper { +public: + using SubArchive = T; + InputArchiveWrapper(const std::string& dir): dir_(dir) { + } + + virtual ~InputArchiveWrapper() { + + } + + size_t load_meta() { + size_t subcnt = 0; + auto ar = std::make_shared(dir_ + "/meta.dump"); + typename SubArchive::Guard guard(ar.get()); + ar->load(&subcnt); + return subcnt; + } + + std::shared_ptr create_archive(size_t i) { + std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump"; + return std::make_shared(file_path); + } +private: + std::string dir_; +}; + } // namespace phmap