diff --git a/examples/serialize.cc b/examples/serialize.cc index a9adcf4..0d44109 100644 --- a/examples/serialize.cc +++ b/examples/serialize.cc @@ -5,8 +5,6 @@ #define USE_CEREAL 0 #if USE_CEREAL - #define PHMAP_DISABLE_DUMP // this is needed because the cereal doesn't like our hash map to have a dump() function - #include "cereal/types/unordered_map.hpp" #include "cereal/types/memory.hpp" #include "cereal/types/bitset.hpp" @@ -53,7 +51,7 @@ public: { return permuteQPR((permuteQPR(m_index++) + m_intermediateOffset) ^ 0x5bf03635); } -}; +}; // -------------------------------------------------------------------------- // -------------------------------------------------------------------------- @@ -68,15 +66,17 @@ void showtime(const char *name, std::function doit) // -------------------------------------------------------------------------- // -------------------------------------------------------------------------- -int main() +template +void testMapSerialization(const char *maptype, const char *fname) { - using MapType = phmap::flat_hash_map; MapType table; const int num_items = 100000000; + printf("Building test %s\n", maptype); + // Iterate and add keys and values // ------------------------------- - showtime("build hash", [&table, num_items]() { + showtime("build time", [&table, num_items]() { unsigned int seed = 76687; RSU rsu(seed, seed + 1); @@ -87,12 +87,12 @@ int main() // cerealize and save data // ----------------------- - showtime("serialize", [&table]() { + showtime("serialize", [&]() { #if !USE_CEREAL - phmap::BinaryOutputArchive ar_out("./dump.data"); - table.dump(ar_out); + phmap::BinaryOutputArchive ar_out(fname); + table.phmap_dump(ar_out); #else - ofstream os("out.cereal", ios::binary); + ofstream os(fname, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); cereal::BinaryOutputArchive archive(os); archive(table.size()); archive(table); @@ -103,12 +103,12 @@ int main() // deserialize // ----------- - showtime("deserialize", [&table_in]() { + showtime("deserialize", [&]() { #if !USE_CEREAL - phmap::BinaryInputArchive ar_in("./dump.data"); - table_in.load(ar_in); + phmap::BinaryInputArchive ar_in(fname); + table_in.phmap_load(ar_in); #else - ifstream is("out.cereal", ios::binary); + ifstream is(fname, std::ofstream::in | std::ofstream::binary); cereal::BinaryInputArchive archive_in(is); size_t table_size; @@ -120,9 +120,85 @@ int main() if (table == table_in) - printf("All checks out, table size: %zu\n", table_in.size()); + printf("All checks out, table size: %zu\n\n", table_in.size()); else printf("FAILURE\n"); +} + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template +void testSetSerialization(const char *settype, const char *fname) +{ + SetType table; + const int num_items = 100000000; + + printf("Building test %s\n", settype); + + // Iterate and add keys and values + // ------------------------------- + showtime("build time", [&]() { + unsigned int seed = 76687; + RSU rsu(seed, seed + 1); + + table.reserve(num_items); + for (int i=0; i < num_items; ++i) + table.insert(typename SetType::value_type(rsu.next())); + }); + + // cerealize and save data + // ----------------------- + showtime("serialize", [&]() { +#if !USE_CEREAL + phmap::BinaryOutputArchive ar_out(fname); + table.phmap_dump(ar_out); +#else + ofstream os(fname, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); + cereal::BinaryOutputArchive archive(os); + archive(table.size()); + archive(table); +#endif + }); + + SetType table_in; + + // deserialize + // ----------- + showtime("deserialize", [&]() { +#if !USE_CEREAL + phmap::BinaryInputArchive ar_in(fname); + table_in.phmap_load(ar_in); +#else + ifstream is(fname, std::ofstream::in | std::ofstream::binary); + cereal::BinaryInputArchive archive_in(is); + size_t table_size; + + archive_in(table_size); + table_in.reserve(table_size); + archive_in(table_in); // deserialize from file out.cereal into table_in +#endif + }); + + + if (table == table_in) + printf("All checks out, table size: %zu\n\n", table_in.size()); + else + printf("FAILURE\n"); +} + + + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +int main() +{ + testSetSerialization>("flat_hash_set", "dump1.bin"); +#if 0 + testSetSerialization>("parallel_flat_hash_set", "dump1.bin"); + + testMapSerialization>("flat_hash_map", "dump1.bin"); + testMapSerialization>("parallel_flat_hash_map", "dump1.bin"); +#endif return 0; } diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 2736290..f8de239 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1577,12 +1577,12 @@ public: } } -#if !defined(PHMAP_NON_DETERMINISTIC) && !defined(PHMAP_DISABLE_DUMP) +#if !defined(PHMAP_NON_DETERMINISTIC) template - bool dump(OutputArchive&) const; + bool phmap_dump(OutputArchive&) const; template - bool load(InputArchive&); + bool phmap_load(InputArchive&); #endif void rehash(size_t n) { @@ -3367,12 +3367,12 @@ public: return HashElement{hash_ref()}(key); } -#if !defined(PHMAP_NON_DETERMINISTIC) && !defined(PHMAP_DISABLE_DUMP) +#if !defined(PHMAP_NON_DETERMINISTIC) template - bool dump(OutputArchive& ar) const; + bool phmap_dump(OutputArchive& ar) const; template - bool load(InputArchive& ar); + bool phmap_load(InputArchive& ar); #endif private: diff --git a/parallel_hashmap/phmap_dump.h b/parallel_hashmap/phmap_dump.h index 38af55c..a4b8a59 100644 --- a/parallel_hashmap/phmap_dump.h +++ b/parallel_hashmap/phmap_dump.h @@ -49,65 +49,34 @@ namespace priv { // ------------------------------------------------------------------------ template template -bool raw_hash_set::dump(OutputArchive& ar) const { +bool raw_hash_set::phmap_dump(OutputArchive& ar) const { static_assert(type_traits_internal::IsTriviallyCopyable::value, "value_type should be trivially copyable"); - if (!ar.dump(size_)) { - std::cerr << "Failed to dump size_" << std::endl; - return false; - } - if (size_ == 0) { + ar.saveBinary(&size_, sizeof(size_t)); + if (size_ == 0) return true; - } - if (!ar.dump(capacity_)) { - std::cerr << "Failed to dump capacity_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { - - std::cerr << "Failed to dump ctrl_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast(slots_), - sizeof(slot_type) * capacity_)) { - std::cerr << "Failed to dump slot_" << std::endl; - return false; - } + ar.saveBinary(&capacity_, sizeof(size_t)); + ar.saveBinary(ctrl_, sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1)); + ar.saveBinary(slots_, sizeof(slot_type) * capacity_); return true; } template template -bool raw_hash_set::load(InputArchive& ar) { +bool raw_hash_set::phmap_load(InputArchive& ar) { static_assert(type_traits_internal::IsTriviallyCopyable::value, "value_type should be trivially copyable"); raw_hash_set().swap(*this); // clear any existing content - if (!ar.load(&size_)) { - std::cerr << "Failed to load size_" << std::endl; - return false; - } - if (size_ == 0) { + ar.loadBinary(&size_, sizeof(size_t)); + if (size_ == 0) return true; - } - if (!ar.load(&capacity_)) { - std::cerr << "Failed to load capacity_" << std::endl; - return false; - } + ar.loadBinary(&capacity_, sizeof(size_t)); // allocate memory for ctrl_ and slots_ initialize_slots(capacity_); - if (!ar.load(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { - std::cerr << "Failed to load ctrl" << std::endl; - return false; - } - if (!ar.load(reinterpret_cast(slots_), - sizeof(slot_type) * capacity_)) { - std::cerr << "Failed to load slot" << std::endl; - return false; - } + ar.loadBinary(ctrl_, sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1)); + ar.loadBinary(slots_, sizeof(slot_type) * capacity_); return true; } @@ -119,18 +88,16 @@ template template -bool parallel_hash_set::dump(OutputArchive& ar) const { +bool parallel_hash_set::phmap_dump(OutputArchive& ar) const { static_assert(type_traits_internal::IsTriviallyCopyable::value, - "value_type should be trivially copyable"); + "value_type should be trivially copyable"); - if (! ar.dump(subcnt())) { - std::cerr << "Failed to dump meta!" << std::endl; - return false; - } + size_t submap_count = subcnt(); + ar.saveBinary(&submap_count, sizeof(size_t)); for (size_t i = 0; i < sets_.size(); ++i) { auto& inner = sets_[i]; typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.dump(ar)) { + if (!inner.set_.phmap_dump(ar)) { std::cerr << "Failed to dump submap " << i << std::endl; return false; } @@ -143,16 +110,12 @@ template template -bool parallel_hash_set::load(InputArchive& ar) { +bool parallel_hash_set::phmap_load(InputArchive& ar) { static_assert(type_traits_internal::IsTriviallyCopyable::value, - "value_type should be trivially copyable"); + "value_type should be trivially copyable"); size_t submap_count = 0; - if (!ar.load(&submap_count)) { - std::cerr << "Failed to load submap count!" << std::endl; - return false; - } - + ar.loadBinary(&submap_count, sizeof(size_t)); if (submap_count != subcnt()) { std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; return false; @@ -161,7 +124,7 @@ bool parallel_hash_set::load(InputArch for (size_t i = 0; i < submap_count; ++i) { auto& inner = sets_[i]; typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.load(ar)) { + if (!inner.set_.phmap_load(ar)) { std::cerr << "Failed to load submap " << i << std::endl; return false; } @@ -185,18 +148,11 @@ bool parallel_hash_set::load(InputArch class BinaryOutputArchive { public: BinaryOutputArchive(const char *file_path) { - ofs_.open(file_path, std::ios_base::binary); + ofs_.open(file_path, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); } - bool dump(const char *p, size_t sz) { - ofs_.write(p, sz); - return true; - } - - template - typename std::enable_if::value, bool>::type - dump(const V& v) { - ofs_.write(reinterpret_cast(&v), sizeof(V)); + bool saveBinary(const void *p, size_t sz) { + ofs_.write(reinterpret_cast(p), sz); return true; } @@ -208,18 +164,11 @@ private: class BinaryInputArchive { public: BinaryInputArchive(const char * file_path) { - ifs_.open(file_path, std::ios_base::binary); + ifs_.open(file_path, std::ofstream::in | std::ofstream::binary); } - bool load(char* p, size_t sz) { - ifs_.read(p, sz); - return true; - } - - template - typename std::enable_if::value, bool>::type - load(V* v) { - ifs_.read(reinterpret_cast(v), sizeof(V)); + bool loadBinary(void* p, size_t sz) { + ifs_.read(reinterpret_cast(p), sz); return true; } @@ -229,4 +178,83 @@ private: } // namespace phmap + +#ifdef CEREAL_SIZE_TYPE + +template +using PhmapTrivCopyable = typename phmap::type_traits_internal::IsTriviallyCopyable; + +namespace cereal +{ + // Overload Cereal serialization code for phmap::flat_hash_map + // ----------------------------------------------------------- + template + void save(typename std::enable_if::value && PhmapTrivCopyable::value, typename cereal::BinaryOutputArchive>::type &ar, + phmap::flat_hash_map const &hmap) + { + hmap.phmap_dump(ar); + } + + template + void load(typename std::enable_if::value && PhmapTrivCopyable::value, typename cereal::BinaryInputArchive>::type &ar, + phmap::flat_hash_map &hmap) + { + hmap.phmap_load(ar); + } + + + // Overload Cereal serialization code for phmap::parallel_flat_hash_map + // -------------------------------------------------------------------- + template + void save(typename std::enable_if::value && PhmapTrivCopyable::value, typename cereal::BinaryOutputArchive>::type &ar, + phmap::parallel_flat_hash_map const &hmap) + { + hmap.phmap_dump(ar); + } + + template + void load(typename std::enable_if::value && PhmapTrivCopyable::value, typename cereal::BinaryInputArchive>::type &ar, + phmap::parallel_flat_hash_map &hmap) + { + hmap.phmap_load(ar); + } + + // Overload Cereal serialization code for phmap::flat_hash_set + // ----------------------------------------------------------- + template + void save(typename std::enable_if::value, typename cereal::BinaryOutputArchive>::type &ar, + phmap::flat_hash_set const &hset) + { + hset.phmap_dump(ar); + } + + template + void load(typename std::enable_if::value, typename cereal::BinaryInputArchive>::type &ar, + phmap::flat_hash_set &hset) + { + hset.phmap_load(ar); + } + + // Overload Cereal serialization code for phmap::parallel_flat_hash_set + // -------------------------------------------------------------------- + template + void save(typename std::enable_if::value, typename cereal::BinaryOutputArchive>::type &ar, + phmap::parallel_flat_hash_set const &hset) + { + hset.phmap_dump(ar); + } + + template + void load(typename std::enable_if::value, typename cereal::BinaryInputArchive>::type &ar, + phmap::parallel_flat_hash_set &hset) + { + hset.phmap_load(ar); + } +} + +#endif + + + + #endif // phmap_dump_h_guard_