From 5cd20dc4a69fbe68cba2fdcbd708ea0a654bf52b Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Thu, 15 Aug 2019 12:23:05 +0800 Subject: [PATCH] added serialize compare and added parallel dump&load --- CMakeLists.txt | 1 + examples/dump_load.cc | 48 +++++++++---- examples/serialize.cc | 4 +- examples/serialize_compare.cc | 130 ++++++++++++++++++++++++++++++++++ parallel_hashmap/phmap.h | 65 ++++++++++++++++- 5 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 examples/serialize_compare.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 12e46c5..5b3bcae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,6 +132,7 @@ if (PHMAP_BUILD_EXAMPLES) add_executable(ex_insert_bench examples/insert_bench.cc phmap.natvis) add_executable(ex_knucleotide examples/knucleotide.cc phmap.natvis) add_executable(ex_dump_load examples/dump_load.cc phmap.natvis) + # add_executable(ex_serialize_compare examples/serialize_compare.cc phmap.natvis) target_link_libraries(ex_knucleotide Threads::Threads) target_link_libraries(ex_bench Threads::Threads) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index a6c64cf..8f652ff 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -4,8 +4,9 @@ using phmap::flat_hash_map; using phmap::flat_hash_set; +using phmap::parallel_flat_hash_map; -void load_dump_string_string() { +void dump_load_string_string() { flat_hash_map mp1; // Add a new entry @@ -27,7 +28,7 @@ void load_dump_string_string() { std::remove("./dump.data"); } -void load_dump_uint64_uint32() { +void dump_load_uint64_uint32() { flat_hash_map mp1; // Add a new entry @@ -49,7 +50,7 @@ void load_dump_uint64_uint32() { std::remove("./dump.data"); } -void load_dump_string_uint32() { +void dump_load_string_uint32() { flat_hash_map mp1; // Add a new entry @@ -71,7 +72,7 @@ void load_dump_string_uint32() { std::remove("./dump.data"); } -void load_dump_uint32_string() { +void dump_load_uint32_string() { flat_hash_map mp1; // Add a new entry @@ -93,7 +94,7 @@ void load_dump_uint32_string() { std::remove("./dump.data"); } -void load_dump_string() { +void dump_load_string() { flat_hash_set st1; // Add a new entry @@ -115,7 +116,7 @@ void load_dump_string() { std::remove("./dump.data"); } -void load_dump_uint64() { +void dump_load_uint64() { flat_hash_set st1; // Add a new entry @@ -137,13 +138,36 @@ void load_dump_uint64() { std::remove("./dump.data"); } +void dump_load_parallel_flat_hash_map() { + parallel_flat_hash_map mp1; + + // Add a new entry + mp1[100] = 99; + mp1[300] = 299; + mp1[101] = 992; + mp1[1300] = 2991; + mp1[1130] = 299; + mp1[2130] = 1299; + // Iterate and print + for (const auto& n : mp1) + std::cout << "key: " << n.first << ", value: " << n.second << "\n"; + + mp1.dump("./dump"); + parallel_flat_hash_map mp2; + + mp2.load("./dump"); + for (const auto& n : mp2) + std::cout << "key: " << n.first << ", value: " << n.second << "\n"; +} + int main() { - load_dump_string_string(); - load_dump_uint64_uint32(); - load_dump_string_uint32(); - load_dump_uint32_string(); - load_dump_string(); - load_dump_uint64(); + dump_load_string_string(); + dump_load_uint64_uint32(); + dump_load_string_uint32(); + dump_load_uint32_string(); + dump_load_string(); + dump_load_uint64(); + dump_load_parallel_flat_hash_map(); return 0; } \ No newline at end of file diff --git a/examples/serialize.cc b/examples/serialize.cc index 5b49b7a..9867ce6 100644 --- a/examples/serialize.cc +++ b/examples/serialize.cc @@ -22,9 +22,9 @@ void showtime(const char *name, std::function doit) doit(); auto t2 = std::chrono::high_resolution_clock::now(); auto elapsed = milliseconds(t2 - t1).count(); - printf("%s: %.3fms\n", name, (int)elapsed / 1000.0f); + printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f); } - + int main() { using MapType = phmap::flat_hash_map, int>; diff --git a/examples/serialize_compare.cc b/examples/serialize_compare.cc new file mode 100644 index 0000000..1cd7986 --- /dev/null +++ b/examples/serialize_compare.cc @@ -0,0 +1,130 @@ +#include +#include +#include +#include "parallel_hashmap/phmap.h" +#include "cereal/types/unordered_map.hpp" +#include "cereal/types/memory.hpp" +#include "cereal/types/bitset.hpp" +#include "cereal/archives/binary.hpp" +#include +#include +#include +#include +#include + +using phmap::flat_hash_map; +using namespace std; +template using milliseconds = std::chrono::duration; + +void showtime(const char *name, std::function doit) +{ + auto t1 = std::chrono::high_resolution_clock::now(); + doit(); + auto t2 = std::chrono::high_resolution_clock::now(); + auto elapsed = milliseconds(t2 - t1).count(); + printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f); +} + +const size_t bigger_than_cachesize = 10 * 1024 * 1024; +long *p = new long[bigger_than_cachesize]; +long *p1 = new long[bigger_than_cachesize]; + +int main(int argc, char* argv[]) +{ + size_t num_items = 2000000; + if (argc == 2) { + num_items = atoi(argv[1]); + } + std::cout << "items size: " << num_items << std::endl; + + using MapType = phmap::flat_hash_map; + MapType table; + + std::vector test_data_arr(num_items, 0); + for (size_t i = 0; i < num_items; i++) { + test_data_arr[i] = i; + } + + std::srand(std::time(nullptr)); + auto generate_random = [](int n) -> size_t { + return std::rand() % n; + }; + + auto shuffle = [&generate_random](std::vector& arr) { + size_t n = arr.size(); + size_t pos = n; + while(pos > 0) { + size_t r = generate_random(pos); + swap(arr[r], arr[pos - 1]); + pos --; + } + }; + + // shuffle + shuffle(test_data_arr); + + // Iterate and add keys and values + // ------------------------------- + showtime("build hash", [&table, &test_data_arr, num_items]() { + table.reserve(num_items); + for (int i=0; i < num_items; ++i) { + table[test_data_arr[i]] = rand(); + } + }); + + // cerealize and save data + // ----------------------- + showtime("serialize", [&table]() { + ofstream os("out.cereal", ios::binary); + cereal::BinaryOutputArchive archive(os); + archive(table.size()); + archive(table); + }); + + MapType().swap(table); // make sure table is newly created + + + // "flush" cache. + for(int i = 0; i < bigger_than_cachesize; i++) { + p[i] = rand(); + } + + // deserialize + // ----------- + showtime("deserialize", [&table]() { + ifstream is("out.cereal", ios::binary); + cereal::BinaryInputArchive archive_in(is); + size_t table_size; + + archive_in(table_size); + table.reserve(table_size); + archive_in(table); // deserialize from file out.cereal into variable + }); + + + printf("table size: %zu\n", table.size()); + + + // dump data + // ----------------------- + showtime("dump", [&table]() { + table.dump("./out.dump"); + }); + + // "flush" cache. + for(int i = 0; i < bigger_than_cachesize; i++) { + p1[i] = rand(); + } + + + MapType().swap(table); // make sure table is newly created + + // load + // ----------- + showtime("load", [&table]() { + table.load("./out.dump"); + }); + + printf("table size: %zu\n", table.size()); + return 0; +} diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index e715734..34e7552 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1554,7 +1554,7 @@ public: return false; } ofs.write(reinterpret_cast(&size_), sizeof(size_)); - ofs.write(reinterpret_cast(&capacity_), sizeof(capacity_)); + ofs.write(reinterpret_cast(&capacity_), sizeof(capacity_)); ofs.write(reinterpret_cast(ctrl_), capacity_ * sizeof(ctrl_t)); ofs.write(reinterpret_cast(slots_), capacity_ * sizeof(slot_type)); ofs.close(); @@ -3287,6 +3287,69 @@ public: a.swap(b); } + bool dump(const std::string& dump_dir) { + for (size_t i = 0; i < sets_.size(); ++i) { + auto& inner = sets_[i]; + if (inner.set_.size() == 0) { + continue; + } + const std::string& dump_path = dump_dir + "/submap_" + std::to_string(i) + ".dump"; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.dump(dump_path)) { + return false; + } + } + std::ofstream fout(dump_dir + "/dump.meta"); + fout << sets_.size(); // submap count + fout.close(); + return true; + } + + bool load(const std::string& load_dir) { + std::ifstream fin(load_dir + "/dump.meta"); + if (! fin.is_open()) { + std::cerr << "Failed to find dump.meta in dir " << load_dir << std::endl; + return false; + } + + size_t submap_count = 0; + fin >> submap_count; + if (submap_count <= 0) { + std::cerr << "Invalid submap count: " << submap_count << std::endl; + return false; + } + + fin.close(); + + if (submap_count != subcnt()) { + std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; + return false; + } + + auto file_exists = [] (const std::string& file_name) -> bool { + std::ifstream fin(file_name); + if (fin.is_open()) { + fin.close(); + return true; + } else { + fin.close(); + return false; + } + }; + + for (size_t i = 0; i < submap_count; ++i) { + auto& inner = sets_[i]; + const std::string& load_file = load_dir + "/submap_" + std::to_string(i) + ".dump"; + if (!file_exists(load_file)) { + continue; + } + if (!inner.set_.load(load_file)) { + return false; + } + } + return true; + } + private: template friend struct phmap::container_internal::hashtable_debug_internal::HashtableDebugAccess;