added serialize compare and added parallel dump&load

CMakeLists.txt

@@ -132,6 +132,7 @@ if (PHMAP_BUILD_EXAMPLES)
     add_executable(ex_insert_bench examples/insert_bench.cc phmap.natvis)
     add_executable(ex_knucleotide examples/knucleotide.cc phmap.natvis)
     add_executable(ex_dump_load examples/dump_load.cc phmap.natvis)
+    # add_executable(ex_serialize_compare examples/serialize_compare.cc phmap.natvis)
 
     target_link_libraries(ex_knucleotide Threads::Threads)
     target_link_libraries(ex_bench Threads::Threads)

examples/dump_load.cc

@@ -4,8 +4,9 @@
 
 using phmap::flat_hash_map;
 using phmap::flat_hash_set;
+using phmap::parallel_flat_hash_map;
 
-void load_dump_string_string() {
+void dump_load_string_string() {
     flat_hash_map<std::string, std::string> mp1;
  
     // Add a new entry
@@ -27,7 +28,7 @@ void load_dump_string_string() {
     std::remove("./dump.data");
 }
 
-void load_dump_uint64_uint32() {
+void dump_load_uint64_uint32() {
     flat_hash_map<uint64_t, uint32_t> mp1;
  
     // Add a new entry
@@ -49,7 +50,7 @@ void load_dump_uint64_uint32() {
     std::remove("./dump.data");
 }
 
-void load_dump_string_uint32() {
+void dump_load_string_uint32() {
     flat_hash_map<std::string, uint32_t> mp1;
  
     // Add a new entry
@@ -71,7 +72,7 @@ void load_dump_string_uint32() {
     std::remove("./dump.data");
 }
 
-void load_dump_uint32_string() {
+void dump_load_uint32_string() {
     flat_hash_map<uint32_t, std::string> mp1;
  
     // Add a new entry
@@ -93,7 +94,7 @@ void load_dump_uint32_string() {
     std::remove("./dump.data");
 }
 
-void load_dump_string() {
+void dump_load_string() {
     flat_hash_set<std::string> st1;
  
     // Add a new entry
@@ -115,7 +116,7 @@ void load_dump_string() {
     std::remove("./dump.data");
 }
 
-void load_dump_uint64() {
+void dump_load_uint64() {
     flat_hash_set<uint64_t> st1;
  
     // Add a new entry
@@ -137,13 +138,36 @@ void load_dump_uint64() {
     std::remove("./dump.data");
 }
 
+void dump_load_parallel_flat_hash_map() {
+    parallel_flat_hash_map<uint64_t, uint32_t> mp1;
+ 
+    // Add a new entry
+    mp1[100] = 99;
+    mp1[300] = 299;
+    mp1[101] = 992;
+    mp1[1300] = 2991;
+    mp1[1130] = 299;
+    mp1[2130] = 1299;
+    // Iterate and print
+    for (const auto& n : mp1)
+        std::cout << "key: " << n.first << ", value: " << n.second << "\n";
+ 
+    mp1.dump("./dump");
+    parallel_flat_hash_map<uint64_t, uint32_t> mp2;
+
+    mp2.load("./dump");
+    for (const auto& n : mp2)
+        std::cout << "key: " << n.first << ", value: " << n.second << "\n";
+}
+
 int main()
 {    
-    load_dump_string_string();
+    dump_load_string_string();
-    load_dump_uint64_uint32();
+    dump_load_uint64_uint32();
-    load_dump_string_uint32();
+    dump_load_string_uint32();
-    load_dump_uint32_string();
+    dump_load_uint32_string();
-    load_dump_string();
+    dump_load_string();
-    load_dump_uint64();
+    dump_load_uint64();
+    dump_load_parallel_flat_hash_map();
     return 0;
 }

examples/serialize.cc

@@ -22,7 +22,7 @@ void showtime(const char *name, std::function<void ()> doit)
     doit();
     auto t2 = std::chrono::high_resolution_clock::now();
     auto elapsed = milliseconds<double>(t2 - t1).count();
-    printf("%s: %.3fms\n", name, (int)elapsed / 1000.0f);
+    printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f);
 }
 
 int main()

examples/serialize_compare.cc

@@ -0,0 +1,130 @@
+#include <iostream>
+#include <bitset>
+#include <cinttypes>
+#include "parallel_hashmap/phmap.h"
+#include "cereal/types/unordered_map.hpp"
+#include "cereal/types/memory.hpp"
+#include "cereal/types/bitset.hpp"
+#include "cereal/archives/binary.hpp"
+#include <fstream>
+#include <random>
+#include <chrono>
+#include <functional>
+#include <cstdio>
+
+using phmap::flat_hash_map;
+using namespace std;
+template <typename T> using milliseconds = std::chrono::duration<T, std::milli>;
+
+void showtime(const char *name, std::function<void ()> doit)
+{
+    auto t1 = std::chrono::high_resolution_clock::now();
+    doit();
+    auto t2 = std::chrono::high_resolution_clock::now();
+    auto elapsed = milliseconds<double>(t2 - t1).count();
+    printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f);
+}
+
+const size_t bigger_than_cachesize = 10 * 1024 * 1024;
+long *p = new long[bigger_than_cachesize];
+long *p1 = new long[bigger_than_cachesize];
+
+int main(int argc, char* argv[])
+{
+    size_t num_items =  2000000;
+    if (argc == 2) {
+        num_items = atoi(argv[1]);
+    }
+    std::cout << "items size: " << num_items << std::endl;
+
+    using MapType = phmap::flat_hash_map<uint64_t, int>;
+    MapType table;
+
+    std::vector<uint64_t> test_data_arr(num_items, 0);
+    for (size_t i = 0; i < num_items; i++) {
+        test_data_arr[i] = i;
+    }
+
+    std::srand(std::time(nullptr));
+    auto generate_random = [](int n) -> size_t {
+        return std::rand() % n;
+    };
+
+    auto shuffle = [&generate_random](std::vector<uint64_t>& arr) {
+        size_t n = arr.size();
+        size_t pos = n;
+        while(pos > 0) {
+            size_t r = generate_random(pos);
+            swap(arr[r], arr[pos - 1]);
+            pos --;
+        }
+    };
+
+    // shuffle
+    shuffle(test_data_arr);
+
+    // Iterate and add keys and values
+    // -------------------------------
+    showtime("build hash", [&table, &test_data_arr, num_items]() {
+            table.reserve(num_items);
+            for (int i=0; i < num_items; ++i) {
+                table[test_data_arr[i]] = rand();
+            }
+        });
+
+    // cerealize and save data
+    // -----------------------
+    showtime("serialize", [&table]() {
+            ofstream os("out.cereal", ios::binary);
+            cereal::BinaryOutputArchive archive(os);
+            archive(table.size());
+            archive(table);
+        });
+
+    MapType().swap(table); // make sure table is newly created
+
+
+    // "flush" cache.
+    for(int i = 0; i < bigger_than_cachesize; i++) {
+        p[i] = rand();
+    }
+
+    // deserialize
+    // -----------
+    showtime("deserialize", [&table]() {
+            ifstream is("out.cereal", ios::binary);
+            cereal::BinaryInputArchive archive_in(is);
+            size_t table_size;
+
+            archive_in(table_size);
+            table.reserve(table_size);
+            archive_in(table);             // deserialize from file out.cereal into variable
+        });
+
+
+    printf("table size: %zu\n", table.size());
+
+
+    // dump data
+    // -----------------------
+    showtime("dump", [&table]() {
+            table.dump("./out.dump");
+        });
+
+    // "flush" cache.
+    for(int i = 0; i < bigger_than_cachesize; i++) {
+        p1[i] = rand();
+    }
+
+
+    MapType().swap(table); // make sure table is newly created
+
+    // load
+    // -----------
+    showtime("load", [&table]() {
+            table.load("./out.dump");
+        });
+
+    printf("table size: %zu\n", table.size());
+    return 0;
+}

parallel_hashmap/phmap.h

@@ -3287,6 +3287,69 @@ public:
         a.swap(b);
     }
 
+    bool dump(const std::string& dump_dir) {
+        for (size_t i = 0; i <  sets_.size(); ++i) {
+            auto& inner = sets_[i];
+            if (inner.set_.size() == 0) {
+                continue;
+            }
+            const std::string& dump_path = dump_dir + "/submap_" + std::to_string(i) + ".dump";
+            typename Lockable::UniqueLock m(const_cast<Inner&>(inner));
+            if (!inner.set_.dump(dump_path)) {
+                return false;
+            }
+        }
+        std::ofstream fout(dump_dir + "/dump.meta");
+        fout << sets_.size(); // submap count
+        fout.close();
+        return true;
+    }
+
+    bool load(const std::string& load_dir) {
+        std::ifstream fin(load_dir + "/dump.meta");
+        if (! fin.is_open()) {
+            std::cerr << "Failed to find dump.meta in dir " << load_dir << std::endl;
+            return false;
+        }
+
+        size_t submap_count = 0;
+        fin >> submap_count;
+        if (submap_count <= 0) {
+            std::cerr << "Invalid submap count: " << submap_count << std::endl;
+            return false;
+        }
+
+        fin.close();
+
+        if (submap_count != subcnt()) {
+            std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl;
+            return false;
+        }
+
+        auto file_exists = [] (const std::string& file_name) -> bool {
+            std::ifstream fin(file_name);
+            if (fin.is_open()) {
+                fin.close();
+                return true;
+            } else {
+                fin.close();
+                return false;
+            }
+        };
+
+        for (size_t i = 0; i < submap_count; ++i) {
+            auto& inner = sets_[i];
+            const std::string& load_file = load_dir + "/submap_" + std::to_string(i) + ".dump";
+            if (!file_exists(load_file)) {
+                continue;
+            }
+            if (!inner.set_.load(load_file)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
 private:
     template <class Container, typename Enabler>
     friend struct phmap::container_internal::hashtable_debug_internal::HashtableDebugAccess;