diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b3bcae..15017bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,10 @@ if (PHMAP_BUILD_TESTS) phmap_cc_test(NAME parallel_flat_hash_map_mutex SRCS "tests/parallel_flat_hash_map_mutex_test.cc" COPTS "-DUNORDERED_MAP_CXX17" DEPS gmock_main) + phmap_cc_test(NAME dump_load SRCS "tests/dump_load_test.cc" + COPTS "-DUNORDERED_MAP_CXX17" DEPS gmock_main) + + endif() if (PHMAP_BUILD_EXAMPLES) @@ -132,7 +136,6 @@ if (PHMAP_BUILD_EXAMPLES) add_executable(ex_insert_bench examples/insert_bench.cc phmap.natvis) add_executable(ex_knucleotide examples/knucleotide.cc phmap.natvis) add_executable(ex_dump_load examples/dump_load.cc phmap.natvis) - # add_executable(ex_serialize_compare examples/serialize_compare.cc phmap.natvis) target_link_libraries(ex_knucleotide Threads::Threads) target_link_libraries(ex_bench Threads::Threads) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index 8c20d52..8a77e6c 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -3,33 +3,8 @@ #include using phmap::flat_hash_map; -using phmap::flat_hash_set; using phmap::parallel_flat_hash_map; -void dump_load_string_string() { - flat_hash_map mp1; - phmap::BinaryOutputArchive ar_out("./dump.data"); - // Add a new entry - mp1["key-1"] = "value-1"; - mp1["key-2"] = "value-2"; - - // Iterate and print keys and values - for (const auto& n : mp1) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - mp1.dump(ar_out); - flat_hash_map mp2; - - phmap::BinaryInputArchive ar_in("./dump.data"); - - mp2.load(ar_in); - // Iterate and print keys and values g|++ - for (const auto& n : mp2) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - std::remove("./dump.data"); -} - void dump_load_uint64_uint32() { flat_hash_map mp1; phmap::BinaryOutputArchive ar_out("./dump.data"); @@ -48,103 +23,6 @@ void dump_load_uint64_uint32() { // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; - - std::remove("./dump.data"); -} - -void dump_load_string_uint32() { - flat_hash_map mp1; - phmap::BinaryOutputArchive ar_out("./dump.data"); - // Add a new entry - mp1["key-1"] = 99; - mp1["key-2"] = 299; - - // Iterate and print keys and values - for (const auto& n : mp1) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - mp1.dump(ar_out); - flat_hash_map mp2; - phmap::BinaryInputArchive ar_in("./dump.data"); - - mp2.load(ar_in); - // Iterate and print keys and values g|++ - for (const auto& n : mp2) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - std::remove("./dump.data"); -} - -void dump_load_uint32_string() { - flat_hash_map mp1; - phmap::BinaryOutputArchive ar_out("./dump.data"); - - // Add a new entry - mp1[100] = "hello"; - mp1[299] = "world"; - - // Iterate and print keys and values - for (const auto& n : mp1) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - mp1.dump(ar_out); - flat_hash_map mp2; - phmap::BinaryInputArchive ar_in("./dump.data"); - - mp2.load(ar_in); - // Iterate and print keys and values g|++ - for (const auto& n : mp2) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - std::remove("./dump.data"); -} - -void dump_load_string() { - flat_hash_set st1; - phmap::BinaryOutputArchive ar_out("./dump.data"); - - // Add a new entry - st1.insert("hello"); - st1.insert("world"); - - // Iterate and print - for (const auto& n : st1) - std::cout << "value: " << n << "\n"; - - st1.dump(ar_out); - flat_hash_set st2; - phmap::BinaryInputArchive ar_in("./dump.data"); - - st2.load(ar_in); - // Iterate and print keys and values g|++ - for (const auto& n : st2) - std::cout << "value: " << n << "\n"; - - std::remove("./dump.data"); -} - -void dump_load_uint64() { - flat_hash_set st1; - phmap::BinaryOutputArchive ar_out("./dump.data"); - - // Add a new entry - st1.insert(878); - st1.insert(1424); - - // Iterate and print - for (const auto& n : st1) - std::cout << "value: " << n << "\n"; - - st1.dump(ar_out); - flat_hash_set st2; - phmap::BinaryInputArchive ar_in("./dump.data"); - - st2.load(ar_in); - // Iterate and print keys and values g|++ - for (const auto& n : st2) - std::cout << "value: " << n << "\n"; - - std::remove("./dump.data"); } void dump_load_parallel_flat_hash_map() { @@ -172,13 +50,8 @@ void dump_load_parallel_flat_hash_map() { } int main() -{ - dump_load_string_string(); +{ dump_load_uint64_uint32(); - dump_load_string_uint32(); - dump_load_uint32_string(); - dump_load_string(); - dump_load_uint64(); dump_load_parallel_flat_hash_map(); return 0; } \ No newline at end of file diff --git a/examples/serialize_compare.cc b/examples/serialize_compare.cc deleted file mode 100644 index 1cd7986..0000000 --- a/examples/serialize_compare.cc +++ /dev/null @@ -1,130 +0,0 @@ -#include -#include -#include -#include "parallel_hashmap/phmap.h" -#include "cereal/types/unordered_map.hpp" -#include "cereal/types/memory.hpp" -#include "cereal/types/bitset.hpp" -#include "cereal/archives/binary.hpp" -#include -#include -#include -#include -#include - -using phmap::flat_hash_map; -using namespace std; -template using milliseconds = std::chrono::duration; - -void showtime(const char *name, std::function doit) -{ - auto t1 = std::chrono::high_resolution_clock::now(); - doit(); - auto t2 = std::chrono::high_resolution_clock::now(); - auto elapsed = milliseconds(t2 - t1).count(); - printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f); -} - -const size_t bigger_than_cachesize = 10 * 1024 * 1024; -long *p = new long[bigger_than_cachesize]; -long *p1 = new long[bigger_than_cachesize]; - -int main(int argc, char* argv[]) -{ - size_t num_items = 2000000; - if (argc == 2) { - num_items = atoi(argv[1]); - } - std::cout << "items size: " << num_items << std::endl; - - using MapType = phmap::flat_hash_map; - MapType table; - - std::vector test_data_arr(num_items, 0); - for (size_t i = 0; i < num_items; i++) { - test_data_arr[i] = i; - } - - std::srand(std::time(nullptr)); - auto generate_random = [](int n) -> size_t { - return std::rand() % n; - }; - - auto shuffle = [&generate_random](std::vector& arr) { - size_t n = arr.size(); - size_t pos = n; - while(pos > 0) { - size_t r = generate_random(pos); - swap(arr[r], arr[pos - 1]); - pos --; - } - }; - - // shuffle - shuffle(test_data_arr); - - // Iterate and add keys and values - // ------------------------------- - showtime("build hash", [&table, &test_data_arr, num_items]() { - table.reserve(num_items); - for (int i=0; i < num_items; ++i) { - table[test_data_arr[i]] = rand(); - } - }); - - // cerealize and save data - // ----------------------- - showtime("serialize", [&table]() { - ofstream os("out.cereal", ios::binary); - cereal::BinaryOutputArchive archive(os); - archive(table.size()); - archive(table); - }); - - MapType().swap(table); // make sure table is newly created - - - // "flush" cache. - for(int i = 0; i < bigger_than_cachesize; i++) { - p[i] = rand(); - } - - // deserialize - // ----------- - showtime("deserialize", [&table]() { - ifstream is("out.cereal", ios::binary); - cereal::BinaryInputArchive archive_in(is); - size_t table_size; - - archive_in(table_size); - table.reserve(table_size); - archive_in(table); // deserialize from file out.cereal into variable - }); - - - printf("table size: %zu\n", table.size()); - - - // dump data - // ----------------------- - showtime("dump", [&table]() { - table.dump("./out.dump"); - }); - - // "flush" cache. - for(int i = 0; i < bigger_than_cachesize; i++) { - p1[i] = rand(); - } - - - MapType().swap(table); // make sure table is newly created - - // load - // ----------- - showtime("load", [&table]() { - table.load("./out.dump"); - }); - - printf("table size: %zu\n", table.size()); - return 0; -} diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 175a758..de42395 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1552,14 +1552,16 @@ public: std::cerr << "Failed to dump capacity_" << std::endl; return false; } - if (!ar.dump(reinterpret_cast(ctrl_), sizeof(ctrl_t) * capacity_)) { + if (!ar.dump(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { + std::cerr << "Failed to dump ctrl_" << std::endl; return false; } if (!ar.dump(reinterpret_cast(slots_), sizeof(slot_type) * capacity_)) { std::cerr << "Failed to dump slot_" << std::endl; return false; - } + } return true; } @@ -1580,7 +1582,8 @@ public: } // allocate memory for ctrl_ and slots_ initialize_slots(); - if (!ar.load(reinterpret_cast(ctrl_), sizeof(ctrl_t) * capacity_)) { + if (!ar.load(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { std::cerr << "Failed to load ctrl" << std::endl; return false; } diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc new file mode 100644 index 0000000..8424a5d --- /dev/null +++ b/tests/dump_load_test.cc @@ -0,0 +1,105 @@ +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "parallel_hashmap/phmap.h" + +namespace phmap { +namespace container_internal { +namespace { +using ::phmap::flat_hash_set; +using ::phmap::flat_hash_map; +using ::phmap::parallel_flat_hash_map; +using ::phmap::BinaryOutputArchive; +using ::phmap::BinaryInputArchive; +using ::phmap::OutputArchiveWrapper; +using ::phmap::InputArchiveWrapper; + +TEST(DumpLoad, FlatHashSet_string) { + flat_hash_set st1; + BinaryOutputArchive ar_out("./dump.data"); + + st1.insert("hello"); + st1.insert("world"); + + EXPECT_TRUE(st1.dump(ar_out)); + flat_hash_set st2; + BinaryInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(st2.load(ar_in)); + + EXPECT_EQ(2, st2.size()); + EXPECT_TRUE(st2.count("hello")); + EXPECT_TRUE(st2.count("world")); +} + +TEST(DumpLoad, FlatHashMap_string_uint32) { + flat_hash_map mp1; + BinaryOutputArchive ar_out("./dump.data"); + + mp1["key-1"] = 99; + mp1["key-2"] = 299; + + EXPECT_TRUE(mp1.dump(ar_out)); + flat_hash_map mp2; + BinaryInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(mp2.load(ar_in)); + + EXPECT_EQ(2, mp2.size()); + + EXPECT_TRUE(mp2.count("key-1")); + EXPECT_TRUE(mp2.count("key-2")); + EXPECT_EQ(99, mp2.at("key-1")); + EXPECT_EQ(299, mp2.at("key-2")); +} + +TEST(DumpLoad, FlatHashMap_uint64_uint32) { + flat_hash_map mp1; + BinaryOutputArchive ar_out("./dump.data"); + + mp1[78731] = 99; + mp1[13141] = 299; + mp1[2651] = 101; + + EXPECT_TRUE(mp1.dump(ar_out)); + flat_hash_map mp2; + BinaryInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(mp2.load(ar_in)); + + EXPECT_EQ(3, mp2.size()); + EXPECT_TRUE(mp2.count(78731)); + EXPECT_TRUE(mp2.count(13141)); + EXPECT_EQ(99, mp2.at(78731)); + EXPECT_EQ(101, mp2.at(2651)); +} + +TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { + parallel_flat_hash_map mp1; + OutputArchiveWrapper w_out("./"); + + mp1[100] = 99; + mp1[300] = 299; + mp1[101] = 992; + mp1[1300] = 2991; + mp1[1130] = 299; + mp1[2130] = 1299; + + EXPECT_TRUE(mp1.dump(w_out)); + parallel_flat_hash_map mp2; + InputArchiveWrapper w_in("./"); + + EXPECT_TRUE(mp2.load(w_in)); + + EXPECT_EQ(6, mp2.size()); + EXPECT_EQ(99, mp2[100]); + EXPECT_EQ(299, mp2[300]); + EXPECT_EQ(299, mp2[1130]); + EXPECT_EQ(1299, mp2[2130]); +} + +} +} +} \ No newline at end of file