Add binary dump/load cereal support for flat hash maps - Issue #120

examples/serialize.cc

@@ -5,8 +5,6 @@
 #define USE_CEREAL 0
 
 #if USE_CEREAL
-    #define PHMAP_DISABLE_DUMP // this is needed because the cereal doesn't like our hash map to have a dump() function
-
     #include "cereal/types/unordered_map.hpp"
     #include "cereal/types/memory.hpp"
     #include "cereal/types/bitset.hpp"
@@ -53,7 +51,7 @@ public:
     {
         return permuteQPR((permuteQPR(m_index++) + m_intermediateOffset) ^ 0x5bf03635);
     }
-};
+};  
 
 // --------------------------------------------------------------------------
 // --------------------------------------------------------------------------
@@ -68,15 +66,17 @@ void showtime(const char *name, std::function<void ()> doit)
 
 // --------------------------------------------------------------------------
 // --------------------------------------------------------------------------
-int main()
+template <class MapType>
+void testMapSerialization(const char *maptype, const char *fname)
 {
-    using MapType = phmap::flat_hash_map<unsigned int, int>;
     MapType table;
     const int num_items = 100000000;
 
+    printf("Building test %s\n", maptype);
+
     // Iterate and add keys and values 
     // -------------------------------
-    showtime("build hash", [&table, num_items]() {
+    showtime("build time", [&table, num_items]() {
             unsigned int seed = 76687;
             RSU rsu(seed, seed + 1);
 
@@ -87,12 +87,12 @@ int main()
 
     // cerealize and save data
     // -----------------------
-    showtime("serialize", [&table]() {
+    showtime("serialize", [&]() {
 #if !USE_CEREAL
-            phmap::BinaryOutputArchive ar_out("./dump.data");
+            phmap::BinaryOutputArchive ar_out(fname);
-            table.dump(ar_out);
+            table.phmap_dump(ar_out);
 #else
-            ofstream os("out.cereal", ios::binary);
+            ofstream os(fname, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary);
             cereal::BinaryOutputArchive archive(os);
             archive(table.size());
             archive(table);
@@ -103,12 +103,12 @@ int main()
 
     // deserialize
     // -----------
-    showtime("deserialize", [&table_in]() {
+    showtime("deserialize", [&]() {
 #if !USE_CEREAL
-            phmap::BinaryInputArchive ar_in("./dump.data");
+            phmap::BinaryInputArchive ar_in(fname);
-            table_in.load(ar_in);
+            table_in.phmap_load(ar_in);
 #else
-            ifstream is("out.cereal", ios::binary);
+            ifstream is(fname, std::ofstream::in | std::ofstream::binary);
             cereal::BinaryInputArchive archive_in(is);
             size_t table_size;
 
@@ -120,9 +120,85 @@ int main()
 
     
     if (table == table_in)
-        printf("All checks out, table size: %zu\n", table_in.size());
+        printf("All checks out, table size: %zu\n\n", table_in.size());
     else
         printf("FAILURE\n");
+}
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+template <class SetType>
+void testSetSerialization(const char *settype, const char *fname)
+{
+    SetType table;
+    const int num_items = 100000000;
+
+    printf("Building test %s\n", settype);
+
+    // Iterate and add keys and values 
+    // -------------------------------
+    showtime("build time", [&]() {
+            unsigned int seed = 76687;
+            RSU rsu(seed, seed + 1);
+
+            table.reserve(num_items);
+            for (int i=0; i < num_items; ++i) 
+                table.insert(typename SetType::value_type(rsu.next())); 
+        });
+
+    // cerealize and save data
+    // -----------------------
+    showtime("serialize", [&]() {
+#if !USE_CEREAL
+            phmap::BinaryOutputArchive ar_out(fname);
+            table.phmap_dump(ar_out);
+#else
+            ofstream os(fname, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary);
+            cereal::BinaryOutputArchive archive(os);
+            archive(table.size());
+            archive(table);
+#endif
+        });
+
+    SetType table_in;
+
+    // deserialize
+    // -----------
+    showtime("deserialize", [&]() {
+#if !USE_CEREAL
+            phmap::BinaryInputArchive ar_in(fname);
+            table_in.phmap_load(ar_in);
+#else
+            ifstream is(fname, std::ofstream::in | std::ofstream::binary);
+            cereal::BinaryInputArchive archive_in(is);
+            size_t table_size;
+
+            archive_in(table_size);
+            table_in.reserve(table_size);
+            archive_in(table_in);             // deserialize from file out.cereal into table_in
+#endif
+        });
+
+    
+    if (table == table_in)
+        printf("All checks out, table size: %zu\n\n", table_in.size());
+    else
+        printf("FAILURE\n");
+}
+
+
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+int main()
+{
+    testSetSerialization<phmap::flat_hash_set<unsigned int>>("flat_hash_set", "dump1.bin");
+#if 0
+    testSetSerialization<phmap::parallel_flat_hash_set<unsigned int>>("parallel_flat_hash_set", "dump1.bin");
+
+    testMapSerialization<phmap::flat_hash_map<unsigned int, int>>("flat_hash_map", "dump1.bin");
+    testMapSerialization<phmap::parallel_flat_hash_map<unsigned int, int>>("parallel_flat_hash_map", "dump1.bin");
+#endif
 
     return 0;
 }

parallel_hashmap/phmap.h

@@ -1577,12 +1577,12 @@ public:
         }
     }
 
-#if !defined(PHMAP_NON_DETERMINISTIC) && !defined(PHMAP_DISABLE_DUMP)
+#if !defined(PHMAP_NON_DETERMINISTIC)
     template<typename OutputArchive>
-    bool dump(OutputArchive&) const;
+    bool phmap_dump(OutputArchive&) const;
 
     template<typename InputArchive>
-    bool load(InputArchive&);
+    bool  phmap_load(InputArchive&);
 #endif
 
     void rehash(size_t n) {
@@ -3367,12 +3367,12 @@ public:
         return HashElement{hash_ref()}(key);
     }
 
-#if !defined(PHMAP_NON_DETERMINISTIC) && !defined(PHMAP_DISABLE_DUMP)
+#if !defined(PHMAP_NON_DETERMINISTIC)
     template<typename OutputArchive>
-    bool dump(OutputArchive& ar) const;
+    bool phmap_dump(OutputArchive& ar) const;
 
     template<typename InputArchive>
-    bool load(InputArchive& ar);
+    bool phmap_load(InputArchive& ar);
 #endif
 
 private:

parallel_hashmap/phmap_dump.h

@@ -49,65 +49,34 @@ namespace priv {
 // ------------------------------------------------------------------------
 template <class Policy, class Hash, class Eq, class Alloc>
 template<typename OutputArchive>
-bool raw_hash_set<Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) const {
+bool raw_hash_set<Policy, Hash, Eq, Alloc>::phmap_dump(OutputArchive& ar) const {
     static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
                     "value_type should be trivially copyable");
 
-    if (!ar.dump(size_)) {
+    ar.saveBinary(&size_, sizeof(size_t));
-        std::cerr << "Failed to dump size_" << std::endl;
+    if (size_ == 0)
-        return false;
-    }
-    if (size_ == 0) {
         return true;
-    }
+    ar.saveBinary(&capacity_, sizeof(size_t));
-    if (!ar.dump(capacity_)) {
+    ar.saveBinary(ctrl_,  sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1));
-        std::cerr << "Failed to dump capacity_" << std::endl;
+    ar.saveBinary(slots_, sizeof(slot_type) * capacity_);
-        return false;
-    }
-    if (!ar.dump(reinterpret_cast<char*>(ctrl_),
-        sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) {
-
-        std::cerr << "Failed to dump ctrl_" << std::endl;
-        return false;
-    }
-    if (!ar.dump(reinterpret_cast<char*>(slots_),
-                    sizeof(slot_type) * capacity_)) {
-        std::cerr << "Failed to dump slot_" << std::endl;
-        return false;
-    }
     return true;
 }
 
 template <class Policy, class Hash, class Eq, class Alloc>
 template<typename InputArchive>
-bool raw_hash_set<Policy, Hash, Eq, Alloc>::load(InputArchive& ar) {
+bool raw_hash_set<Policy, Hash, Eq, Alloc>::phmap_load(InputArchive& ar) {
     static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
                     "value_type should be trivially copyable");
     raw_hash_set<Policy, Hash, Eq, Alloc>().swap(*this); // clear any existing content
-    if (!ar.load(&size_)) {
+    ar.loadBinary(&size_, sizeof(size_t));
-        std::cerr << "Failed to load size_" << std::endl;
+    if (size_ == 0)
-        return false;
-    }
-    if (size_ == 0) {
         return true;
-    }
+    ar.loadBinary(&capacity_, sizeof(size_t));
-    if (!ar.load(&capacity_)) {
-        std::cerr << "Failed to load capacity_" << std::endl;
-        return false;
-    }
 
     // allocate memory for ctrl_ and slots_
     initialize_slots(capacity_);
-    if (!ar.load(reinterpret_cast<char*>(ctrl_),
+    ar.loadBinary(ctrl_,  sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1));
-        sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) {
+    ar.loadBinary(slots_, sizeof(slot_type) * capacity_);
-        std::cerr << "Failed to load ctrl" << std::endl;
-        return false;
-    }
-    if (!ar.load(reinterpret_cast<char*>(slots_),
-                    sizeof(slot_type) * capacity_)) {
-        std::cerr << "Failed to load slot" << std::endl;
-        return false;
-    }
     return true;
 }
 
@@ -119,18 +88,16 @@ template <size_t N,
           class Mtx_,
           class Policy, class Hash, class Eq, class Alloc>
 template<typename OutputArchive>
-bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) const {
+bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::phmap_dump(OutputArchive& ar) const {
     static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
-                    "value_type should be trivially copyable");
+                  "value_type should be trivially copyable");
 
-    if (! ar.dump(subcnt())) {
+    size_t submap_count = subcnt();
-        std::cerr << "Failed to dump meta!" << std::endl;
+    ar.saveBinary(&submap_count, sizeof(size_t));
-        return false;
-    }
     for (size_t i = 0; i < sets_.size(); ++i) {
         auto& inner = sets_[i];
         typename Lockable::UniqueLock m(const_cast<Inner&>(inner));
-        if (!inner.set_.dump(ar)) {
+        if (!inner.set_.phmap_dump(ar)) {
             std::cerr << "Failed to dump submap " << i << std::endl;
             return false;
         }
@@ -143,16 +110,12 @@ template <size_t N,
           class Mtx_,
           class Policy, class Hash, class Eq, class Alloc>
 template<typename InputArchive>
-bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::load(InputArchive& ar) {
+bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::phmap_load(InputArchive& ar) {
     static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
-                    "value_type should be trivially copyable");
+                  "value_type should be trivially copyable");
 
     size_t submap_count = 0;
-    if (!ar.load(&submap_count)) {
+    ar.loadBinary(&submap_count, sizeof(size_t));
-        std::cerr << "Failed to load submap count!" << std::endl;
-        return false;
-    }
-
     if (submap_count != subcnt()) {
         std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl;
         return false;
@@ -161,7 +124,7 @@ bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::load(InputArch
     for (size_t i = 0; i < submap_count; ++i) {            
         auto& inner = sets_[i];
         typename Lockable::UniqueLock m(const_cast<Inner&>(inner));
-        if (!inner.set_.load(ar)) {
+        if (!inner.set_.phmap_load(ar)) {
             std::cerr << "Failed to load submap " << i << std::endl;
             return false;
         }
@@ -185,18 +148,11 @@ bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::load(InputArch
 class BinaryOutputArchive {
 public:
     BinaryOutputArchive(const char *file_path) {
-        ofs_.open(file_path, std::ios_base::binary);
+        ofs_.open(file_path, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary);
     }
 
-    bool dump(const char *p, size_t sz) {
+    bool saveBinary(const void *p, size_t sz) {
-        ofs_.write(p, sz);
+        ofs_.write(reinterpret_cast<const char*>(p), sz);
-        return true;
-    }
-
-    template<typename V>
-    typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type
-    dump(const V& v) {
-        ofs_.write(reinterpret_cast<const char *>(&v), sizeof(V));
         return true;
     }
 
@@ -208,18 +164,11 @@ private:
 class BinaryInputArchive {
 public:
     BinaryInputArchive(const char * file_path) {
-        ifs_.open(file_path, std::ios_base::binary);
+        ifs_.open(file_path, std::ofstream::in | std::ofstream::binary);
     }
 
-    bool load(char* p, size_t sz) {
+    bool loadBinary(void* p, size_t sz) {
-        ifs_.read(p, sz);
+        ifs_.read(reinterpret_cast<char*>(p), sz);
-        return true;
-    }
-
-    template<typename V>
-    typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type
-    load(V* v) {
-        ifs_.read(reinterpret_cast<char *>(v), sizeof(V));
         return true;
     }
 
@@ -229,4 +178,83 @@ private:
 
 } // namespace phmap
 
+
+#ifdef CEREAL_SIZE_TYPE
+
+template <class T>
+using PhmapTrivCopyable = typename phmap::type_traits_internal::IsTriviallyCopyable<T>;
+    
+namespace cereal
+{
+    // Overload Cereal serialization code for phmap::flat_hash_map
+    // -----------------------------------------------------------
+    template <class K, class V, class Hash, class Eq, class A>
+    void save(typename std::enable_if<PhmapTrivCopyable<K>::value && PhmapTrivCopyable<V>::value, typename cereal::BinaryOutputArchive>::type &ar,
+              phmap::flat_hash_map<K, V, Hash, Eq, A> const &hmap)
+    {
+        hmap.phmap_dump(ar);
+    }
+
+    template <class K, class V, class Hash, class Eq, class A>
+    void load(typename std::enable_if<PhmapTrivCopyable<K>::value && PhmapTrivCopyable<V>::value, typename cereal::BinaryInputArchive>::type &ar, 
+              phmap::flat_hash_map<K, V, Hash, Eq, A>  &hmap)
+    {
+        hmap.phmap_load(ar);
+    }
+
+
+    // Overload Cereal serialization code for phmap::parallel_flat_hash_map
+    // --------------------------------------------------------------------
+    template <class K, class V, class Hash, class Eq, class A, size_t N, class Mtx_>
+    void save(typename std::enable_if<PhmapTrivCopyable<K>::value && PhmapTrivCopyable<V>::value, typename cereal::BinaryOutputArchive>::type &ar,
+              phmap::parallel_flat_hash_map<K, V, Hash, Eq, A, N, Mtx_> const &hmap)
+    {
+        hmap.phmap_dump(ar);
+    }
+
+    template <class K, class V, class Hash, class Eq, class A, size_t N, class Mtx_>
+    void load(typename std::enable_if<PhmapTrivCopyable<K>::value && PhmapTrivCopyable<V>::value, typename cereal::BinaryInputArchive>::type &ar, 
+              phmap::parallel_flat_hash_map<K, V, Hash, Eq, A, N, Mtx_>  &hmap)
+    {
+        hmap.phmap_load(ar);
+    }
+
+    // Overload Cereal serialization code for phmap::flat_hash_set
+    // -----------------------------------------------------------
+    template <class K, class Hash, class Eq, class A>
+    void save(typename std::enable_if<PhmapTrivCopyable<K>::value, typename cereal::BinaryOutputArchive>::type &ar,
+              phmap::flat_hash_set<K, Hash, Eq, A> const &hset)
+    {
+        hset.phmap_dump(ar);
+    }
+
+    template <class K, class Hash, class Eq, class A>
+    void load(typename std::enable_if<PhmapTrivCopyable<K>::value, typename cereal::BinaryInputArchive>::type &ar, 
+              phmap::flat_hash_set<K, Hash, Eq, A>  &hset)
+    {
+        hset.phmap_load(ar);
+    }
+
+    // Overload Cereal serialization code for phmap::parallel_flat_hash_set
+    // --------------------------------------------------------------------
+    template <class K, class Hash, class Eq, class A, size_t N, class Mtx_>
+    void save(typename std::enable_if<PhmapTrivCopyable<K>::value, typename cereal::BinaryOutputArchive>::type &ar,
+              phmap::parallel_flat_hash_set<K, Hash, Eq, A, N, Mtx_> const &hset)
+    {
+        hset.phmap_dump(ar);
+    }
+
+    template <class K, class Hash, class Eq, class A, size_t N, class Mtx_>
+    void load(typename std::enable_if<PhmapTrivCopyable<K>::value, typename cereal::BinaryInputArchive>::type &ar, 
+              phmap::parallel_flat_hash_set<K, Hash, Eq, A, N, Mtx_>  &hset)
+    {
+        hset.phmap_load(ar);
+    }
+}
+
+#endif
+
+
+
+
 #endif // phmap_dump_h_guard_