From 5e5ade8c6e404742f89e4e9271f5062d87372f4f Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Tue, 13 Aug 2019 22:42:37 +0800 Subject: [PATCH 01/13] added load&dump interface --- .gitignore | 1 + CMakeLists.txt | 3 + examples/dump_load.cc | 138 +++++++++++++++++++++++++++++++ parallel_hashmap/phmap.h | 148 ++++++++++++++++++++++++++++++++++ parallel_hashmap/phmap_base.h | 82 ++++++++++++++++++- 5 files changed, 371 insertions(+), 1 deletion(-) create mode 100644 examples/dump_load.cc diff --git a/.gitignore b/.gitignore index c09a4b2..a6d3cf8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ benchmark/charts.html build build_linux .vagrant +**/.vscode TAGS diff --git a/CMakeLists.txt b/CMakeLists.txt index 072802e..12e46c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ set(CMAKE_SUPPRESS_REGENERATION true) ## suppress ZERO_CHECK project include(GNUInstallDirs) include(CMakePackageConfigHelpers) include(helpers) +include_directories("${CMAKE_CURRENT_SOURCE_DIR}") add_library(${PROJECT_NAME} INTERFACE) @@ -130,6 +131,8 @@ if (PHMAP_BUILD_EXAMPLES) add_executable(ex_two_files examples/f1.cc examples/f2.cc phmap.natvis) add_executable(ex_insert_bench examples/insert_bench.cc phmap.natvis) add_executable(ex_knucleotide examples/knucleotide.cc phmap.natvis) + add_executable(ex_dump_load examples/dump_load.cc phmap.natvis) + target_link_libraries(ex_knucleotide Threads::Threads) target_link_libraries(ex_bench Threads::Threads) endif() diff --git a/examples/dump_load.cc b/examples/dump_load.cc new file mode 100644 index 0000000..f4ae0bd --- /dev/null +++ b/examples/dump_load.cc @@ -0,0 +1,138 @@ +#include +#include +#include + +using phmap::flat_hash_map; +using phmap::flat_hash_set; + +void load_dump_string_string() { + flat_hash_map mp1; + + // Add a new entry + mp1["key-1"] = "value-1"; + mp1["key-2"] = "value-2"; + + // Iterate and print keys and values + for (const auto& n : mp1) + std::cout << n.first << "'s value is: " << n.second << "\n"; + + mp1.dump("./dump.data"); + flat_hash_map mp2; + + mp2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : mp2) + std::cout << n.first << "'s value is: " << n.second << "\n"; +} + +void load_dump_uint64_uint32() { + flat_hash_map mp1; + + // Add a new entry + mp1[100] = 99; + mp1[300] = 299; + + // Iterate and print keys and values + for (const auto& n : mp1) + std::cout << n.first << "'s value is: " << n.second << "\n"; + + mp1.dump("./dump.data"); + flat_hash_map mp2; + + mp2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : mp2) + std::cout << n.first << "'s value is: " << n.second << "\n"; +} + +void load_dump_string_uint32() { + flat_hash_map mp1; + + // Add a new entry + mp1["key-1"] = 99; + mp1["key-2"] = 299; + + // Iterate and print keys and values + for (const auto& n : mp1) + std::cout << n.first << "'s value is: " << n.second << "\n"; + + mp1.dump("./dump.data"); + flat_hash_map mp2; + + mp2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : mp2) + std::cout << n.first << "'s value is: " << n.second << "\n"; +} + +void load_dump_uint32_string() { + flat_hash_map mp1; + + // Add a new entry + mp1[100] = "hello"; + mp1[299] = "world"; + + // Iterate and print keys and values + for (const auto& n : mp1) + std::cout << n.first << "'s value is: " << n.second << "\n"; + + mp1.dump("./dump.data"); + flat_hash_map mp2; + + mp2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : mp2) + std::cout << n.first << "'s value is: " << n.second << "\n"; +} + +void load_dump_string() { + flat_hash_set st1; + + // Add a new entry + st1.insert("hello"); + st1.insert("world"); + + // Iterate and print + for (const auto& n : st1) + std::cout << "value: " << n << "\n"; + + st1.dump("./dump.data"); + flat_hash_set st2; + + st2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : st2) + std::cout << "value: " << n << "\n"; +} + +void load_dump_uint64() { + flat_hash_set st1; + + // Add a new entry + st1.insert(878); + st1.insert(1424); + + // Iterate and print + for (const auto& n : st1) + std::cout << "value: " << n << "\n"; + + st1.dump("./dump.data"); + flat_hash_set st2; + + st2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : st2) + std::cout << "value: " << n << "\n"; +} + +int main() +{ + load_dump_string_string(); + load_dump_uint64_uint32(); + load_dump_string_uint32(); + load_dump_uint32_string(); + load_dump_string(); + load_dump_uint64(); + std::remove("./dump.data"); + return 0; +} \ No newline at end of file diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 94ed883..e715734 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -45,6 +45,9 @@ #include #include #include +#include +#include +#include #include "phmap_utils.h" #include "phmap_base.h" @@ -1534,6 +1537,151 @@ public: } } + template + typename std::enable_if::value, bool>::type + dump(const std::string& dump_file) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + if (size_ == 0) { + std::cout << "Empty set, nothing to dump" << std::endl; + return true; + } + assert(slots_ != nullptr); + std::ofstream ofs(dump_file); + if (!ofs.is_open()) { + std::cout << "Failed to open dump file " << dump_file << std::endl; + return false; + } + ofs.write(reinterpret_cast(&size_), sizeof(size_)); + ofs.write(reinterpret_cast(&capacity_), sizeof(capacity_)); + ofs.write(reinterpret_cast(ctrl_), capacity_ * sizeof(ctrl_t)); + ofs.write(reinterpret_cast(slots_), capacity_ * sizeof(slot_type)); + ofs.close(); + return true; + } + + template + typename std::enable_if::value, bool>::type + load(const std::string& load_file) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + std::ifstream ifs(load_file); + if (!ifs.is_open()) { + std::cerr << "Failed to open load file " << load_file << std::endl; + return false; + } + // get file size + ifs.seekg(0, std::ios::end); + size_t file_size = ifs.tellg(); + ifs.seekg(0, std::ios::beg); + if (file_size <= sizeof(size_) + sizeof(capacity_)) { + std::cerr << "Invalid file format. file size: " << file_size << ", size_: " + << size_ << ", capacity_: " << capacity_ << ", slot type size: " + << sizeof(slot_type); + return false; + } + + ifs.read(reinterpret_cast(&size_), sizeof(size_)); + ifs.read(reinterpret_cast(&capacity_), sizeof(capacity_)); + if (file_size != sizeof(size_) + sizeof(capacity_) + capacity_ * sizeof(ctrl_t) + + capacity_ * sizeof(slot_type)) { + std::cerr << "Invalid file format. file size: " << file_size << ", size_: " + << size_ << ", capacity_: " << capacity_ << ", slot type size: " + << sizeof(slot_type); + return false; + } + // allocate memory for ctrl_ and slots_ + initialize_slots(); + + ifs.read(reinterpret_cast(ctrl_), capacity_ * sizeof(ctrl_t)); + ifs.read(reinterpret_cast(slots_), capacity_ * sizeof(slot_type)); + ifs.close(); + return true; + } + + // V will be V for hash_set and std::pair for hash_map + template + typename std::enable_if::value + && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + dump(const std::string& dump_file) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + if (size_ == 0) { + std::cout << "Empty set, nothing to dump" << std::endl; + return true; + } + assert(slots_ != nullptr); + std::ofstream ofs(dump_file); + if (!ofs.is_open()) { + std::cout << "Failed to open dump file " << dump_file << std::endl; + return false; + } + + ofs.write(reinterpret_cast(&size_), sizeof(size_)); + + for (auto it = this->begin(); it != this->end(); ++it) { + type_traits_internal::Archive::dump(*it, &ofs); + } + ofs.close(); + return true; + } + + template + typename std::enable_if::value + && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + load(const std::string& load_file = "") noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + + std::ifstream ifs(load_file); + if (!ifs.is_open()) { + std::cerr << "Failed to open load file " << load_file << std::endl; + return false; + } + + size_t total_count = 0; + ifs.read((char*)&total_count, sizeof(total_count)); + + for (size_t i = 0; i < total_count; i ++) { + if (ifs.eof()) { + std::cerr << "Data is not enough, total_count: " << total_count + << ", meet eof at index: " << i << std::endl; + return false; + } + V v; + type_traits_internal::Archive::load(ifs, &v); + this->insert(v); + } + ifs.close(); + return true; + } + + template + typename std::enable_if::value, bool>::type + dump(const std::string&) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + std::cerr << "Does not support this type now!" << std::endl; + std::abort(); + return false; + } + + template + typename std::enable_if::value, bool>::type + load(const std::string&) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + std::cerr << "Does not support this type now!" << std::endl; + std::abort(); + return false; + } + void rehash(size_t n) { if (n == 0 && capacity_ == 0) return; if (n == 0 && size_ == 0) { diff --git a/parallel_hashmap/phmap_base.h b/parallel_hashmap/phmap_base.h index bbc6712..2045d2a 100644 --- a/parallel_hashmap/phmap_base.h +++ b/parallel_hashmap/phmap_base.h @@ -33,7 +33,6 @@ // See the License for the specific language governing permissions and // limitations under the License. // --------------------------------------------------------------------------- - #include #include #include @@ -71,6 +70,87 @@ struct EqualTo namespace type_traits_internal { +template +struct PairTrait : public std::false_type { + using first_type = typename std::remove_cv::type; + using second_type = typename std::remove_cv::type; +}; + +template +struct PairTrait>: public std::true_type { + using first_type = typename std::remove_cv::type; + using second_type = typename std::remove_cv::type; +}; + +template +struct IsArithmeticType { + static constexpr bool value = std::is_arithmetic::value + || (PairTrait::value && + std::is_arithmetic::first_type>::value + && std::is_arithmetic::second_type>::value); +}; + +template +struct IsStringOrArithmeticType { + static constexpr bool value = IsArithmeticType::value + || std::is_same::value + || (PairTrait::value + && (std::is_arithmetic::first_type>::value + || std::is_same::first_type, std::string>::value) + && (std::is_arithmetic::second_type>::value + || std::is_same::second_type, std::string>::value)); +}; + +// only support std::is_arithmetic or std::string types +template +struct Archive { + template + static typename std::enable_if::value, void>::type + dump(const V& v, std::ofstream* ofs) { + ofs->write(reinterpret_cast(const_cast(&v)), sizeof(V)); + } + + template + static typename std::enable_if::value, void>::type + load(std::ifstream& ifs, V* v) { + ifs.read(reinterpret_cast(v), sizeof(V)); + } + + template + static typename std::enable_if::type>::value, void>::type + dump(const V& v, std::ofstream* ofs) { + uint32_t sz = v.length(); + ofs->write(reinterpret_cast(&sz), sizeof(sz)); + ofs->write(const_cast(v.data()), sz); + } + + template + static typename std::enable_if::type>::value, void>::type + load(std::ifstream& ifs, V* v) { + uint32_t sz = 0; + ifs.read(reinterpret_cast(&sz), sizeof(sz)); + const_cast(v)->resize(sz); + ifs.read(const_cast(v->data()), sz); + } + + template + static typename std::enable_if::value && IsStringOrArithmeticType::value, void>::type + dump(const V& v, std::ofstream* ofs) { + dump::first_type>(v.first, ofs); + dump::second_type>(v.second, ofs); + } + + template + static typename std::enable_if::value && IsStringOrArithmeticType::value, void>::type + load(std::ifstream& ifs, V* v) { + using first_type = typename PairTrait::first_type; + using second_type = typename PairTrait::second_type; + load(ifs, const_cast(&v->first)); + load(ifs, const_cast(&v->second)); + } +}; + + template struct VoidTImpl { using type = void; From da55a69dfa189287659a4af90c89c5ada3c623dc Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Wed, 14 Aug 2019 23:28:33 +0800 Subject: [PATCH 02/13] refine test --- examples/dump_load.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index f4ae0bd..a6c64cf 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -23,6 +23,8 @@ void load_dump_string_string() { // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; + + std::remove("./dump.data"); } void load_dump_uint64_uint32() { @@ -43,6 +45,8 @@ void load_dump_uint64_uint32() { // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; + + std::remove("./dump.data"); } void load_dump_string_uint32() { @@ -63,6 +67,8 @@ void load_dump_string_uint32() { // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; + + std::remove("./dump.data"); } void load_dump_uint32_string() { @@ -83,6 +89,8 @@ void load_dump_uint32_string() { // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; + + std::remove("./dump.data"); } void load_dump_string() { @@ -103,6 +111,8 @@ void load_dump_string() { // Iterate and print keys and values g|++ for (const auto& n : st2) std::cout << "value: " << n << "\n"; + + std::remove("./dump.data"); } void load_dump_uint64() { @@ -123,6 +133,8 @@ void load_dump_uint64() { // Iterate and print keys and values g|++ for (const auto& n : st2) std::cout << "value: " << n << "\n"; + + std::remove("./dump.data"); } int main() @@ -133,6 +145,5 @@ int main() load_dump_uint32_string(); load_dump_string(); load_dump_uint64(); - std::remove("./dump.data"); return 0; } \ No newline at end of file From 5cd20dc4a69fbe68cba2fdcbd708ea0a654bf52b Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Thu, 15 Aug 2019 12:23:05 +0800 Subject: [PATCH 03/13] added serialize compare and added parallel dump&load --- CMakeLists.txt | 1 + examples/dump_load.cc | 48 +++++++++---- examples/serialize.cc | 4 +- examples/serialize_compare.cc | 130 ++++++++++++++++++++++++++++++++++ parallel_hashmap/phmap.h | 65 ++++++++++++++++- 5 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 examples/serialize_compare.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 12e46c5..5b3bcae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,6 +132,7 @@ if (PHMAP_BUILD_EXAMPLES) add_executable(ex_insert_bench examples/insert_bench.cc phmap.natvis) add_executable(ex_knucleotide examples/knucleotide.cc phmap.natvis) add_executable(ex_dump_load examples/dump_load.cc phmap.natvis) + # add_executable(ex_serialize_compare examples/serialize_compare.cc phmap.natvis) target_link_libraries(ex_knucleotide Threads::Threads) target_link_libraries(ex_bench Threads::Threads) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index a6c64cf..8f652ff 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -4,8 +4,9 @@ using phmap::flat_hash_map; using phmap::flat_hash_set; +using phmap::parallel_flat_hash_map; -void load_dump_string_string() { +void dump_load_string_string() { flat_hash_map mp1; // Add a new entry @@ -27,7 +28,7 @@ void load_dump_string_string() { std::remove("./dump.data"); } -void load_dump_uint64_uint32() { +void dump_load_uint64_uint32() { flat_hash_map mp1; // Add a new entry @@ -49,7 +50,7 @@ void load_dump_uint64_uint32() { std::remove("./dump.data"); } -void load_dump_string_uint32() { +void dump_load_string_uint32() { flat_hash_map mp1; // Add a new entry @@ -71,7 +72,7 @@ void load_dump_string_uint32() { std::remove("./dump.data"); } -void load_dump_uint32_string() { +void dump_load_uint32_string() { flat_hash_map mp1; // Add a new entry @@ -93,7 +94,7 @@ void load_dump_uint32_string() { std::remove("./dump.data"); } -void load_dump_string() { +void dump_load_string() { flat_hash_set st1; // Add a new entry @@ -115,7 +116,7 @@ void load_dump_string() { std::remove("./dump.data"); } -void load_dump_uint64() { +void dump_load_uint64() { flat_hash_set st1; // Add a new entry @@ -137,13 +138,36 @@ void load_dump_uint64() { std::remove("./dump.data"); } +void dump_load_parallel_flat_hash_map() { + parallel_flat_hash_map mp1; + + // Add a new entry + mp1[100] = 99; + mp1[300] = 299; + mp1[101] = 992; + mp1[1300] = 2991; + mp1[1130] = 299; + mp1[2130] = 1299; + // Iterate and print + for (const auto& n : mp1) + std::cout << "key: " << n.first << ", value: " << n.second << "\n"; + + mp1.dump("./dump"); + parallel_flat_hash_map mp2; + + mp2.load("./dump"); + for (const auto& n : mp2) + std::cout << "key: " << n.first << ", value: " << n.second << "\n"; +} + int main() { - load_dump_string_string(); - load_dump_uint64_uint32(); - load_dump_string_uint32(); - load_dump_uint32_string(); - load_dump_string(); - load_dump_uint64(); + dump_load_string_string(); + dump_load_uint64_uint32(); + dump_load_string_uint32(); + dump_load_uint32_string(); + dump_load_string(); + dump_load_uint64(); + dump_load_parallel_flat_hash_map(); return 0; } \ No newline at end of file diff --git a/examples/serialize.cc b/examples/serialize.cc index 5b49b7a..9867ce6 100644 --- a/examples/serialize.cc +++ b/examples/serialize.cc @@ -22,9 +22,9 @@ void showtime(const char *name, std::function doit) doit(); auto t2 = std::chrono::high_resolution_clock::now(); auto elapsed = milliseconds(t2 - t1).count(); - printf("%s: %.3fms\n", name, (int)elapsed / 1000.0f); + printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f); } - + int main() { using MapType = phmap::flat_hash_map, int>; diff --git a/examples/serialize_compare.cc b/examples/serialize_compare.cc new file mode 100644 index 0000000..1cd7986 --- /dev/null +++ b/examples/serialize_compare.cc @@ -0,0 +1,130 @@ +#include +#include +#include +#include "parallel_hashmap/phmap.h" +#include "cereal/types/unordered_map.hpp" +#include "cereal/types/memory.hpp" +#include "cereal/types/bitset.hpp" +#include "cereal/archives/binary.hpp" +#include +#include +#include +#include +#include + +using phmap::flat_hash_map; +using namespace std; +template using milliseconds = std::chrono::duration; + +void showtime(const char *name, std::function doit) +{ + auto t1 = std::chrono::high_resolution_clock::now(); + doit(); + auto t2 = std::chrono::high_resolution_clock::now(); + auto elapsed = milliseconds(t2 - t1).count(); + printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f); +} + +const size_t bigger_than_cachesize = 10 * 1024 * 1024; +long *p = new long[bigger_than_cachesize]; +long *p1 = new long[bigger_than_cachesize]; + +int main(int argc, char* argv[]) +{ + size_t num_items = 2000000; + if (argc == 2) { + num_items = atoi(argv[1]); + } + std::cout << "items size: " << num_items << std::endl; + + using MapType = phmap::flat_hash_map; + MapType table; + + std::vector test_data_arr(num_items, 0); + for (size_t i = 0; i < num_items; i++) { + test_data_arr[i] = i; + } + + std::srand(std::time(nullptr)); + auto generate_random = [](int n) -> size_t { + return std::rand() % n; + }; + + auto shuffle = [&generate_random](std::vector& arr) { + size_t n = arr.size(); + size_t pos = n; + while(pos > 0) { + size_t r = generate_random(pos); + swap(arr[r], arr[pos - 1]); + pos --; + } + }; + + // shuffle + shuffle(test_data_arr); + + // Iterate and add keys and values + // ------------------------------- + showtime("build hash", [&table, &test_data_arr, num_items]() { + table.reserve(num_items); + for (int i=0; i < num_items; ++i) { + table[test_data_arr[i]] = rand(); + } + }); + + // cerealize and save data + // ----------------------- + showtime("serialize", [&table]() { + ofstream os("out.cereal", ios::binary); + cereal::BinaryOutputArchive archive(os); + archive(table.size()); + archive(table); + }); + + MapType().swap(table); // make sure table is newly created + + + // "flush" cache. + for(int i = 0; i < bigger_than_cachesize; i++) { + p[i] = rand(); + } + + // deserialize + // ----------- + showtime("deserialize", [&table]() { + ifstream is("out.cereal", ios::binary); + cereal::BinaryInputArchive archive_in(is); + size_t table_size; + + archive_in(table_size); + table.reserve(table_size); + archive_in(table); // deserialize from file out.cereal into variable + }); + + + printf("table size: %zu\n", table.size()); + + + // dump data + // ----------------------- + showtime("dump", [&table]() { + table.dump("./out.dump"); + }); + + // "flush" cache. + for(int i = 0; i < bigger_than_cachesize; i++) { + p1[i] = rand(); + } + + + MapType().swap(table); // make sure table is newly created + + // load + // ----------- + showtime("load", [&table]() { + table.load("./out.dump"); + }); + + printf("table size: %zu\n", table.size()); + return 0; +} diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index e715734..34e7552 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1554,7 +1554,7 @@ public: return false; } ofs.write(reinterpret_cast(&size_), sizeof(size_)); - ofs.write(reinterpret_cast(&capacity_), sizeof(capacity_)); + ofs.write(reinterpret_cast(&capacity_), sizeof(capacity_)); ofs.write(reinterpret_cast(ctrl_), capacity_ * sizeof(ctrl_t)); ofs.write(reinterpret_cast(slots_), capacity_ * sizeof(slot_type)); ofs.close(); @@ -3287,6 +3287,69 @@ public: a.swap(b); } + bool dump(const std::string& dump_dir) { + for (size_t i = 0; i < sets_.size(); ++i) { + auto& inner = sets_[i]; + if (inner.set_.size() == 0) { + continue; + } + const std::string& dump_path = dump_dir + "/submap_" + std::to_string(i) + ".dump"; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.dump(dump_path)) { + return false; + } + } + std::ofstream fout(dump_dir + "/dump.meta"); + fout << sets_.size(); // submap count + fout.close(); + return true; + } + + bool load(const std::string& load_dir) { + std::ifstream fin(load_dir + "/dump.meta"); + if (! fin.is_open()) { + std::cerr << "Failed to find dump.meta in dir " << load_dir << std::endl; + return false; + } + + size_t submap_count = 0; + fin >> submap_count; + if (submap_count <= 0) { + std::cerr << "Invalid submap count: " << submap_count << std::endl; + return false; + } + + fin.close(); + + if (submap_count != subcnt()) { + std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; + return false; + } + + auto file_exists = [] (const std::string& file_name) -> bool { + std::ifstream fin(file_name); + if (fin.is_open()) { + fin.close(); + return true; + } else { + fin.close(); + return false; + } + }; + + for (size_t i = 0; i < submap_count; ++i) { + auto& inner = sets_[i]; + const std::string& load_file = load_dir + "/submap_" + std::to_string(i) + ".dump"; + if (!file_exists(load_file)) { + continue; + } + if (!inner.set_.load(load_file)) { + return false; + } + } + return true; + } + private: template friend struct phmap::container_internal::hashtable_debug_internal::HashtableDebugAccess; From dd9af7f8a5049f5be39b1d202c5e4068a76b5462 Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Fri, 16 Aug 2019 21:24:02 +0800 Subject: [PATCH 04/13] refine code --- examples/dump_load.cc | 53 +++++--- parallel_hashmap/phmap.h | 237 ++++++++++++--------------------- parallel_hashmap/phmap_base.h | 50 ------- parallel_hashmap/phmap_utils.h | 191 +++++++++++++++++++++++++- 4 files changed, 310 insertions(+), 221 deletions(-) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index 8f652ff..8c20d52 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -8,7 +8,7 @@ using phmap::parallel_flat_hash_map; void dump_load_string_string() { flat_hash_map mp1; - + phmap::BinaryOutputArchive ar_out("./dump.data"); // Add a new entry mp1["key-1"] = "value-1"; mp1["key-2"] = "value-2"; @@ -17,10 +17,12 @@ void dump_load_string_string() { for (const auto& n : mp1) std::cout << n.first << "'s value is: " << n.second << "\n"; - mp1.dump("./dump.data"); + mp1.dump(ar_out); flat_hash_map mp2; - mp2.load("./dump.data"); + phmap::BinaryInputArchive ar_in("./dump.data"); + + mp2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; @@ -30,7 +32,7 @@ void dump_load_string_string() { void dump_load_uint64_uint32() { flat_hash_map mp1; - + phmap::BinaryOutputArchive ar_out("./dump.data"); // Add a new entry mp1[100] = 99; mp1[300] = 299; @@ -39,10 +41,10 @@ void dump_load_uint64_uint32() { for (const auto& n : mp1) std::cout << n.first << "'s value is: " << n.second << "\n"; - mp1.dump("./dump.data"); + mp1.dump(ar_out); flat_hash_map mp2; - - mp2.load("./dump.data"); + phmap::BinaryInputArchive ar_in("./dump.data"); + mp2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; @@ -52,7 +54,7 @@ void dump_load_uint64_uint32() { void dump_load_string_uint32() { flat_hash_map mp1; - + phmap::BinaryOutputArchive ar_out("./dump.data"); // Add a new entry mp1["key-1"] = 99; mp1["key-2"] = 299; @@ -61,10 +63,11 @@ void dump_load_string_uint32() { for (const auto& n : mp1) std::cout << n.first << "'s value is: " << n.second << "\n"; - mp1.dump("./dump.data"); + mp1.dump(ar_out); flat_hash_map mp2; + phmap::BinaryInputArchive ar_in("./dump.data"); - mp2.load("./dump.data"); + mp2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; @@ -74,6 +77,7 @@ void dump_load_string_uint32() { void dump_load_uint32_string() { flat_hash_map mp1; + phmap::BinaryOutputArchive ar_out("./dump.data"); // Add a new entry mp1[100] = "hello"; @@ -83,10 +87,11 @@ void dump_load_uint32_string() { for (const auto& n : mp1) std::cout << n.first << "'s value is: " << n.second << "\n"; - mp1.dump("./dump.data"); + mp1.dump(ar_out); flat_hash_map mp2; + phmap::BinaryInputArchive ar_in("./dump.data"); - mp2.load("./dump.data"); + mp2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; @@ -96,7 +101,8 @@ void dump_load_uint32_string() { void dump_load_string() { flat_hash_set st1; - + phmap::BinaryOutputArchive ar_out("./dump.data"); + // Add a new entry st1.insert("hello"); st1.insert("world"); @@ -105,10 +111,11 @@ void dump_load_string() { for (const auto& n : st1) std::cout << "value: " << n << "\n"; - st1.dump("./dump.data"); + st1.dump(ar_out); flat_hash_set st2; + phmap::BinaryInputArchive ar_in("./dump.data"); - st2.load("./dump.data"); + st2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : st2) std::cout << "value: " << n << "\n"; @@ -118,7 +125,8 @@ void dump_load_string() { void dump_load_uint64() { flat_hash_set st1; - + phmap::BinaryOutputArchive ar_out("./dump.data"); + // Add a new entry st1.insert(878); st1.insert(1424); @@ -127,10 +135,11 @@ void dump_load_uint64() { for (const auto& n : st1) std::cout << "value: " << n << "\n"; - st1.dump("./dump.data"); + st1.dump(ar_out); flat_hash_set st2; + phmap::BinaryInputArchive ar_in("./dump.data"); - st2.load("./dump.data"); + st2.load(ar_in); // Iterate and print keys and values g|++ for (const auto& n : st2) std::cout << "value: " << n << "\n"; @@ -140,7 +149,8 @@ void dump_load_uint64() { void dump_load_parallel_flat_hash_map() { parallel_flat_hash_map mp1; - + phmap::OutputArchiveWrapper w_out("./"); + // Add a new entry mp1[100] = 99; mp1[300] = 299; @@ -152,10 +162,11 @@ void dump_load_parallel_flat_hash_map() { for (const auto& n : mp1) std::cout << "key: " << n.first << ", value: " << n.second << "\n"; - mp1.dump("./dump"); + mp1.dump(w_out); parallel_flat_hash_map mp2; + phmap::InputArchiveWrapper w_in("./"); - mp2.load("./dump"); + mp2.load(w_in); for (const auto& n : mp2) std::cout << "key: " << n.first << ", value: " << n.second << "\n"; } diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 34e7552..175a758 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1537,148 +1537,111 @@ public: } } - template + template typename std::enable_if::value, bool>::type - dump(const std::string& dump_file) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - if (size_ == 0) { - std::cout << "Empty set, nothing to dump" << std::endl; - return true; - } - assert(slots_ != nullptr); - std::ofstream ofs(dump_file); - if (!ofs.is_open()) { - std::cout << "Failed to open dump file " << dump_file << std::endl; + dump(OutputArchive& ar) { + typename OutputArchive::Guard guard(&ar); + if (!ar.dump(size_)) { + std::cerr << "Failed to dump size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.dump(capacity_)) { + std::cerr << "Failed to dump capacity_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(ctrl_), sizeof(ctrl_t) * capacity_)) { + std::cerr << "Failed to dump ctrl_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(slots_), sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to dump slot_" << std::endl; return false; } - ofs.write(reinterpret_cast(&size_), sizeof(size_)); - ofs.write(reinterpret_cast(&capacity_), sizeof(capacity_)); - ofs.write(reinterpret_cast(ctrl_), capacity_ * sizeof(ctrl_t)); - ofs.write(reinterpret_cast(slots_), capacity_ * sizeof(slot_type)); - ofs.close(); return true; } - template + template typename std::enable_if::value, bool>::type - load(const std::string& load_file) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - std::ifstream ifs(load_file); - if (!ifs.is_open()) { - std::cerr << "Failed to open load file " << load_file << std::endl; + load(InputArchive& ar) { + typename InputArchive::Guard guard(&ar); + if (!ar.load(&size_)){ + std::cerr << "Failed to load size_" << std::endl; return false; } - // get file size - ifs.seekg(0, std::ios::end); - size_t file_size = ifs.tellg(); - ifs.seekg(0, std::ios::beg); - if (file_size <= sizeof(size_) + sizeof(capacity_)) { - std::cerr << "Invalid file format. file size: " << file_size << ", size_: " - << size_ << ", capacity_: " << capacity_ << ", slot type size: " - << sizeof(slot_type); - return false; + if (size_ == 0) { + return true; } - - ifs.read(reinterpret_cast(&size_), sizeof(size_)); - ifs.read(reinterpret_cast(&capacity_), sizeof(capacity_)); - if (file_size != sizeof(size_) + sizeof(capacity_) + capacity_ * sizeof(ctrl_t) - + capacity_ * sizeof(slot_type)) { - std::cerr << "Invalid file format. file size: " << file_size << ", size_: " - << size_ << ", capacity_: " << capacity_ << ", slot type size: " - << sizeof(slot_type); + if (!ar.load(&capacity_)) { + std::cerr << "Failed to load capacity_" << std::endl; return false; } // allocate memory for ctrl_ and slots_ initialize_slots(); - - ifs.read(reinterpret_cast(ctrl_), capacity_ * sizeof(ctrl_t)); - ifs.read(reinterpret_cast(slots_), capacity_ * sizeof(slot_type)); - ifs.close(); + if (!ar.load(reinterpret_cast(ctrl_), sizeof(ctrl_t) * capacity_)) { + std::cerr << "Failed to load ctrl" << std::endl; + return false; + } + if (!ar.load(reinterpret_cast(slots_), sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to load slot" << std::endl; + return false; + } return true; } // V will be V for hash_set and std::pair for hash_map - template + template typename std::enable_if::value && type_traits_internal::IsStringOrArithmeticType::value, bool>::type - dump(const std::string& dump_file) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - if (size_ == 0) { - std::cout << "Empty set, nothing to dump" << std::endl; + dump(OutputArchive& ar) { + typename OutputArchive::Guard guard(&ar); + if (!ar.template dump(size_)) { + std::cerr << "Failed to dump size" << std::endl; + return false; + } + if (size_ == 0) { return true; } - assert(slots_ != nullptr); - std::ofstream ofs(dump_file); - if (!ofs.is_open()) { - std::cout << "Failed to open dump file " << dump_file << std::endl; - return false; - } - - ofs.write(reinterpret_cast(&size_), sizeof(size_)); - for (auto it = this->begin(); it != this->end(); ++it) { - type_traits_internal::Archive::dump(*it, &ofs); - } - ofs.close(); - return true; - } - - template - typename std::enable_if::value - && type_traits_internal::IsStringOrArithmeticType::value, bool>::type - load(const std::string& load_file = "") noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - - std::ifstream ifs(load_file); - if (!ifs.is_open()) { - std::cerr << "Failed to open load file " << load_file << std::endl; - return false; - } - - size_t total_count = 0; - ifs.read((char*)&total_count, sizeof(total_count)); - - for (size_t i = 0; i < total_count; i ++) { - if (ifs.eof()) { - std::cerr << "Data is not enough, total_count: " << total_count - << ", meet eof at index: " << i << std::endl; + if (!ar.template dump(*it)) { + std::cerr << "Failed to dump element" << std::endl; return false; } - V v; - type_traits_internal::Archive::load(ifs, &v); - this->insert(v); } - ifs.close(); return true; } - template + template + typename std::enable_if::value + && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + load(InputArchive& ar) { + typename InputArchive::Guard guard(&ar); + size_t sz = 0; + ar.template load(&sz); + for (size_t i = 0; i < sz; i ++) { + V v; + if (!ar.template load(&v)) { + std::cerr << "Failed to load element " << i << std::endl; + return false; + } + this->insert(v); + } + return true; + } + + template typename std::enable_if::value, bool>::type - dump(const std::string&) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { + dump(OutputArchive&) { std::cerr << "Does not support this type now!" << std::endl; - std::abort(); return false; } - template + template typename std::enable_if::value, bool>::type - load(const std::string&) noexcept( - IsNoThrowSwappable() && IsNoThrowSwappable() && - (!AllocTraits::propagate_on_container_swap::value || - IsNoThrowSwappable())) { - std::cerr << "Does not support this type now!" << std::endl; - std::abort(); + load(InputArchive&) { + std::cerr << "Does not support this type now!" << std::endl; return false; } @@ -3287,63 +3250,39 @@ public: a.swap(b); } - bool dump(const std::string& dump_dir) { - for (size_t i = 0; i < sets_.size(); ++i) { + template + bool dump(OutputArchiveWrapper& w) { + for (size_t i = 0; i < sets_.size(); ++i) { auto& inner = sets_[i]; - if (inner.set_.size() == 0) { - continue; - } - const std::string& dump_path = dump_dir + "/submap_" + std::to_string(i) + ".dump"; + auto ar = w.create_archive(i); typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.dump(dump_path)) { + if (!inner.set_.dump(*ar)) { + std::cerr << "Failed to dump submap " << i << std::endl; return false; } } - std::ofstream fout(dump_dir + "/dump.meta"); - fout << sets_.size(); // submap count - fout.close(); + + if (! w.dump_meta(subcnt())) { + std::cerr << "Failed to dump meta!" << std::endl; + return false; + } return true; } - bool load(const std::string& load_dir) { - std::ifstream fin(load_dir + "/dump.meta"); - if (! fin.is_open()) { - std::cerr << "Failed to find dump.meta in dir " << load_dir << std::endl; - return false; - } - - size_t submap_count = 0; - fin >> submap_count; - if (submap_count <= 0) { - std::cerr << "Invalid submap count: " << submap_count << std::endl; - return false; - } - - fin.close(); + template + bool load(InputArchiveWrapper& w) { + size_t submap_count = w.load_meta(); if (submap_count != subcnt()) { std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; return false; } - auto file_exists = [] (const std::string& file_name) -> bool { - std::ifstream fin(file_name); - if (fin.is_open()) { - fin.close(); - return true; - } else { - fin.close(); - return false; - } - }; - - for (size_t i = 0; i < submap_count; ++i) { + for (size_t i = 0; i < sets_.size(); ++i) { + auto ar = w.create_archive(i); auto& inner = sets_[i]; - const std::string& load_file = load_dir + "/submap_" + std::to_string(i) + ".dump"; - if (!file_exists(load_file)) { - continue; - } - if (!inner.set_.load(load_file)) { + if (!inner.set_.load(*ar)) { + std::cerr << "Failed to load submap " << i << std::endl; return false; } } diff --git a/parallel_hashmap/phmap_base.h b/parallel_hashmap/phmap_base.h index 2045d2a..8f03623 100644 --- a/parallel_hashmap/phmap_base.h +++ b/parallel_hashmap/phmap_base.h @@ -101,56 +101,6 @@ struct IsStringOrArithmeticType { || std::is_same::second_type, std::string>::value)); }; -// only support std::is_arithmetic or std::string types -template -struct Archive { - template - static typename std::enable_if::value, void>::type - dump(const V& v, std::ofstream* ofs) { - ofs->write(reinterpret_cast(const_cast(&v)), sizeof(V)); - } - - template - static typename std::enable_if::value, void>::type - load(std::ifstream& ifs, V* v) { - ifs.read(reinterpret_cast(v), sizeof(V)); - } - - template - static typename std::enable_if::type>::value, void>::type - dump(const V& v, std::ofstream* ofs) { - uint32_t sz = v.length(); - ofs->write(reinterpret_cast(&sz), sizeof(sz)); - ofs->write(const_cast(v.data()), sz); - } - - template - static typename std::enable_if::type>::value, void>::type - load(std::ifstream& ifs, V* v) { - uint32_t sz = 0; - ifs.read(reinterpret_cast(&sz), sizeof(sz)); - const_cast(v)->resize(sz); - ifs.read(const_cast(v->data()), sz); - } - - template - static typename std::enable_if::value && IsStringOrArithmeticType::value, void>::type - dump(const V& v, std::ofstream* ofs) { - dump::first_type>(v.first, ofs); - dump::second_type>(v.second, ofs); - } - - template - static typename std::enable_if::value && IsStringOrArithmeticType::value, void>::type - load(std::ifstream& ifs, V* v) { - using first_type = typename PairTrait::first_type; - using second_type = typename PairTrait::second_type; - load(ifs, const_cast(&v->first)); - load(ifs, const_cast(&v->second)); - } -}; - - template struct VoidTImpl { using type = void; diff --git a/parallel_hashmap/phmap_utils.h b/parallel_hashmap/phmap_utils.h index ccc078c..3807498 100644 --- a/parallel_hashmap/phmap_utils.h +++ b/parallel_hashmap/phmap_utils.h @@ -23,8 +23,11 @@ #include #include +#include +#include +#include #include "phmap_bits.h" - +#include "phmap_base.h" namespace phmap { @@ -306,6 +309,192 @@ H HashStateBase::combine(H seed, const T& v, const Ts&... vs) using HashState = HashStateBase; + +// ----------------------------------------------------------------------------- + +#define CHECK_FILE(f) { \ + if (!f.is_open()) { \ + std::cerr << "File is not open!" << std::endl; \ + return false; \ + } \ +} + +template +class ArchiveGuard { +public: + ArchiveGuard(Archive* ar): ar_(ar) {}; + ~ArchiveGuard() { + if (ar_) { + ar_->finish(); + } + } +private: + Archive* ar_; +}; + +class BinaryOutputArchive { +public: + using Guard = ArchiveGuard; + + BinaryOutputArchive(const std::string& file_path) { + ofs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~BinaryOutputArchive() { + finish(); + } + + bool dump(char* p, size_t sz) { + CHECK_FILE(ofs_); + ofs_.write(p, sz); + return true; + } + + template + typename std::enable_if::value, bool>::type + dump(const V& v) { + CHECK_FILE(ofs_); + ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); + return true; + } + + template + typename std::enable_if::type>::value, bool>::type + dump(const V& v) { + CHECK_FILE(ofs_); + uint32_t sz = v.length(); + ofs_.write(reinterpret_cast(&sz), sizeof(sz)); + ofs_.write(const_cast(v.data()), sz); + return true; + } + + template + typename std::enable_if::value + && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + dump(const V& v) { + return dump::first_type>(v.first) + && dump::second_type>(v.second); + } + + void finish() { + if (ofs_.is_open()) { + ofs_.close(); + } + } +private: + std::ofstream ofs_; +}; + + +class BinaryInputArchive { +public: + using Guard = ArchiveGuard; + + BinaryInputArchive(const std::string& file_path) { + ifs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~BinaryInputArchive() { + finish(); + } + + bool load(char* p, size_t sz) { + CHECK_FILE(ifs_); + ifs_.read(p, sz); + return true; + } + + template + typename std::enable_if::value, bool>::type + load(V* v) { + CHECK_FILE(ifs_); + ifs_.read(reinterpret_cast(v), sizeof(V)); + return true; + } + + template + typename std::enable_if::type>::value, bool>::type + load(V* v) { + CHECK_FILE(ifs_); + uint32_t sz = 0; + ifs_.read(reinterpret_cast(&sz), sizeof(sz)); + const_cast(v)->resize(sz); + ifs_.read(const_cast(v->data()), sz); + return true; + } + + template + typename std::enable_if::value + && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + load(V* v) { + using first_type = typename type_traits_internal::PairTrait::first_type; + using second_type = typename type_traits_internal::PairTrait::second_type; + return load(const_cast(&v->first)) + && load(const_cast(&v->second)); + } + + void finish() { + if (ifs_.is_open()) { + ifs_.close(); + } + } +private: + std::ifstream ifs_; +}; + +template +class OutputArchiveWrapper { +public: + using SubArchive = T; + OutputArchiveWrapper(const std::string& dir): dir_(dir) { + } + + virtual ~OutputArchiveWrapper() { + + } + + bool dump_meta(size_t subcnt) { + auto ar = std::make_shared(dir_ + "/meta.dump"); + typename SubArchive::Guard guard(ar.get()); + ar->dump(subcnt); + return true; + } + + std::shared_ptr create_archive(size_t i) { + std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump"; + return std::make_shared(file_path); + } +private: + std::string dir_; +}; + +template +class InputArchiveWrapper { +public: + using SubArchive = T; + InputArchiveWrapper(const std::string& dir): dir_(dir) { + } + + virtual ~InputArchiveWrapper() { + + } + + size_t load_meta() { + size_t subcnt = 0; + auto ar = std::make_shared(dir_ + "/meta.dump"); + typename SubArchive::Guard guard(ar.get()); + ar->load(&subcnt); + return subcnt; + } + + std::shared_ptr create_archive(size_t i) { + std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump"; + return std::make_shared(file_path); + } +private: + std::string dir_; +}; + } // namespace phmap From ba121d043ff1bfcf22e3f65530ccfc6d227264cc Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Sun, 18 Aug 2019 22:50:47 +0800 Subject: [PATCH 05/13] added tests --- CMakeLists.txt | 5 +- examples/dump_load.cc | 129 +-------------------------------- examples/serialize_compare.cc | 130 ---------------------------------- parallel_hashmap/phmap.h | 9 ++- tests/dump_load_test.cc | 105 +++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 262 deletions(-) delete mode 100644 examples/serialize_compare.cc create mode 100644 tests/dump_load_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b3bcae..15017bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,10 @@ if (PHMAP_BUILD_TESTS) phmap_cc_test(NAME parallel_flat_hash_map_mutex SRCS "tests/parallel_flat_hash_map_mutex_test.cc" COPTS "-DUNORDERED_MAP_CXX17" DEPS gmock_main) + phmap_cc_test(NAME dump_load SRCS "tests/dump_load_test.cc" + COPTS "-DUNORDERED_MAP_CXX17" DEPS gmock_main) + + endif() if (PHMAP_BUILD_EXAMPLES) @@ -132,7 +136,6 @@ if (PHMAP_BUILD_EXAMPLES) add_executable(ex_insert_bench examples/insert_bench.cc phmap.natvis) add_executable(ex_knucleotide examples/knucleotide.cc phmap.natvis) add_executable(ex_dump_load examples/dump_load.cc phmap.natvis) - # add_executable(ex_serialize_compare examples/serialize_compare.cc phmap.natvis) target_link_libraries(ex_knucleotide Threads::Threads) target_link_libraries(ex_bench Threads::Threads) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index 8c20d52..8a77e6c 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -3,33 +3,8 @@ #include using phmap::flat_hash_map; -using phmap::flat_hash_set; using phmap::parallel_flat_hash_map; -void dump_load_string_string() { - flat_hash_map mp1; - phmap::BinaryOutputArchive ar_out("./dump.data"); - // Add a new entry - mp1["key-1"] = "value-1"; - mp1["key-2"] = "value-2"; - - // Iterate and print keys and values - for (const auto& n : mp1) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - mp1.dump(ar_out); - flat_hash_map mp2; - - phmap::BinaryInputArchive ar_in("./dump.data"); - - mp2.load(ar_in); - // Iterate and print keys and values g|++ - for (const auto& n : mp2) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - std::remove("./dump.data"); -} - void dump_load_uint64_uint32() { flat_hash_map mp1; phmap::BinaryOutputArchive ar_out("./dump.data"); @@ -48,103 +23,6 @@ void dump_load_uint64_uint32() { // Iterate and print keys and values g|++ for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; - - std::remove("./dump.data"); -} - -void dump_load_string_uint32() { - flat_hash_map mp1; - phmap::BinaryOutputArchive ar_out("./dump.data"); - // Add a new entry - mp1["key-1"] = 99; - mp1["key-2"] = 299; - - // Iterate and print keys and values - for (const auto& n : mp1) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - mp1.dump(ar_out); - flat_hash_map mp2; - phmap::BinaryInputArchive ar_in("./dump.data"); - - mp2.load(ar_in); - // Iterate and print keys and values g|++ - for (const auto& n : mp2) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - std::remove("./dump.data"); -} - -void dump_load_uint32_string() { - flat_hash_map mp1; - phmap::BinaryOutputArchive ar_out("./dump.data"); - - // Add a new entry - mp1[100] = "hello"; - mp1[299] = "world"; - - // Iterate and print keys and values - for (const auto& n : mp1) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - mp1.dump(ar_out); - flat_hash_map mp2; - phmap::BinaryInputArchive ar_in("./dump.data"); - - mp2.load(ar_in); - // Iterate and print keys and values g|++ - for (const auto& n : mp2) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - std::remove("./dump.data"); -} - -void dump_load_string() { - flat_hash_set st1; - phmap::BinaryOutputArchive ar_out("./dump.data"); - - // Add a new entry - st1.insert("hello"); - st1.insert("world"); - - // Iterate and print - for (const auto& n : st1) - std::cout << "value: " << n << "\n"; - - st1.dump(ar_out); - flat_hash_set st2; - phmap::BinaryInputArchive ar_in("./dump.data"); - - st2.load(ar_in); - // Iterate and print keys and values g|++ - for (const auto& n : st2) - std::cout << "value: " << n << "\n"; - - std::remove("./dump.data"); -} - -void dump_load_uint64() { - flat_hash_set st1; - phmap::BinaryOutputArchive ar_out("./dump.data"); - - // Add a new entry - st1.insert(878); - st1.insert(1424); - - // Iterate and print - for (const auto& n : st1) - std::cout << "value: " << n << "\n"; - - st1.dump(ar_out); - flat_hash_set st2; - phmap::BinaryInputArchive ar_in("./dump.data"); - - st2.load(ar_in); - // Iterate and print keys and values g|++ - for (const auto& n : st2) - std::cout << "value: " << n << "\n"; - - std::remove("./dump.data"); } void dump_load_parallel_flat_hash_map() { @@ -172,13 +50,8 @@ void dump_load_parallel_flat_hash_map() { } int main() -{ - dump_load_string_string(); +{ dump_load_uint64_uint32(); - dump_load_string_uint32(); - dump_load_uint32_string(); - dump_load_string(); - dump_load_uint64(); dump_load_parallel_flat_hash_map(); return 0; } \ No newline at end of file diff --git a/examples/serialize_compare.cc b/examples/serialize_compare.cc deleted file mode 100644 index 1cd7986..0000000 --- a/examples/serialize_compare.cc +++ /dev/null @@ -1,130 +0,0 @@ -#include -#include -#include -#include "parallel_hashmap/phmap.h" -#include "cereal/types/unordered_map.hpp" -#include "cereal/types/memory.hpp" -#include "cereal/types/bitset.hpp" -#include "cereal/archives/binary.hpp" -#include -#include -#include -#include -#include - -using phmap::flat_hash_map; -using namespace std; -template using milliseconds = std::chrono::duration; - -void showtime(const char *name, std::function doit) -{ - auto t1 = std::chrono::high_resolution_clock::now(); - doit(); - auto t2 = std::chrono::high_resolution_clock::now(); - auto elapsed = milliseconds(t2 - t1).count(); - printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f); -} - -const size_t bigger_than_cachesize = 10 * 1024 * 1024; -long *p = new long[bigger_than_cachesize]; -long *p1 = new long[bigger_than_cachesize]; - -int main(int argc, char* argv[]) -{ - size_t num_items = 2000000; - if (argc == 2) { - num_items = atoi(argv[1]); - } - std::cout << "items size: " << num_items << std::endl; - - using MapType = phmap::flat_hash_map; - MapType table; - - std::vector test_data_arr(num_items, 0); - for (size_t i = 0; i < num_items; i++) { - test_data_arr[i] = i; - } - - std::srand(std::time(nullptr)); - auto generate_random = [](int n) -> size_t { - return std::rand() % n; - }; - - auto shuffle = [&generate_random](std::vector& arr) { - size_t n = arr.size(); - size_t pos = n; - while(pos > 0) { - size_t r = generate_random(pos); - swap(arr[r], arr[pos - 1]); - pos --; - } - }; - - // shuffle - shuffle(test_data_arr); - - // Iterate and add keys and values - // ------------------------------- - showtime("build hash", [&table, &test_data_arr, num_items]() { - table.reserve(num_items); - for (int i=0; i < num_items; ++i) { - table[test_data_arr[i]] = rand(); - } - }); - - // cerealize and save data - // ----------------------- - showtime("serialize", [&table]() { - ofstream os("out.cereal", ios::binary); - cereal::BinaryOutputArchive archive(os); - archive(table.size()); - archive(table); - }); - - MapType().swap(table); // make sure table is newly created - - - // "flush" cache. - for(int i = 0; i < bigger_than_cachesize; i++) { - p[i] = rand(); - } - - // deserialize - // ----------- - showtime("deserialize", [&table]() { - ifstream is("out.cereal", ios::binary); - cereal::BinaryInputArchive archive_in(is); - size_t table_size; - - archive_in(table_size); - table.reserve(table_size); - archive_in(table); // deserialize from file out.cereal into variable - }); - - - printf("table size: %zu\n", table.size()); - - - // dump data - // ----------------------- - showtime("dump", [&table]() { - table.dump("./out.dump"); - }); - - // "flush" cache. - for(int i = 0; i < bigger_than_cachesize; i++) { - p1[i] = rand(); - } - - - MapType().swap(table); // make sure table is newly created - - // load - // ----------- - showtime("load", [&table]() { - table.load("./out.dump"); - }); - - printf("table size: %zu\n", table.size()); - return 0; -} diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 175a758..de42395 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1552,14 +1552,16 @@ public: std::cerr << "Failed to dump capacity_" << std::endl; return false; } - if (!ar.dump(reinterpret_cast(ctrl_), sizeof(ctrl_t) * capacity_)) { + if (!ar.dump(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { + std::cerr << "Failed to dump ctrl_" << std::endl; return false; } if (!ar.dump(reinterpret_cast(slots_), sizeof(slot_type) * capacity_)) { std::cerr << "Failed to dump slot_" << std::endl; return false; - } + } return true; } @@ -1580,7 +1582,8 @@ public: } // allocate memory for ctrl_ and slots_ initialize_slots(); - if (!ar.load(reinterpret_cast(ctrl_), sizeof(ctrl_t) * capacity_)) { + if (!ar.load(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { std::cerr << "Failed to load ctrl" << std::endl; return false; } diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc new file mode 100644 index 0000000..8424a5d --- /dev/null +++ b/tests/dump_load_test.cc @@ -0,0 +1,105 @@ +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "parallel_hashmap/phmap.h" + +namespace phmap { +namespace container_internal { +namespace { +using ::phmap::flat_hash_set; +using ::phmap::flat_hash_map; +using ::phmap::parallel_flat_hash_map; +using ::phmap::BinaryOutputArchive; +using ::phmap::BinaryInputArchive; +using ::phmap::OutputArchiveWrapper; +using ::phmap::InputArchiveWrapper; + +TEST(DumpLoad, FlatHashSet_string) { + flat_hash_set st1; + BinaryOutputArchive ar_out("./dump.data"); + + st1.insert("hello"); + st1.insert("world"); + + EXPECT_TRUE(st1.dump(ar_out)); + flat_hash_set st2; + BinaryInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(st2.load(ar_in)); + + EXPECT_EQ(2, st2.size()); + EXPECT_TRUE(st2.count("hello")); + EXPECT_TRUE(st2.count("world")); +} + +TEST(DumpLoad, FlatHashMap_string_uint32) { + flat_hash_map mp1; + BinaryOutputArchive ar_out("./dump.data"); + + mp1["key-1"] = 99; + mp1["key-2"] = 299; + + EXPECT_TRUE(mp1.dump(ar_out)); + flat_hash_map mp2; + BinaryInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(mp2.load(ar_in)); + + EXPECT_EQ(2, mp2.size()); + + EXPECT_TRUE(mp2.count("key-1")); + EXPECT_TRUE(mp2.count("key-2")); + EXPECT_EQ(99, mp2.at("key-1")); + EXPECT_EQ(299, mp2.at("key-2")); +} + +TEST(DumpLoad, FlatHashMap_uint64_uint32) { + flat_hash_map mp1; + BinaryOutputArchive ar_out("./dump.data"); + + mp1[78731] = 99; + mp1[13141] = 299; + mp1[2651] = 101; + + EXPECT_TRUE(mp1.dump(ar_out)); + flat_hash_map mp2; + BinaryInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(mp2.load(ar_in)); + + EXPECT_EQ(3, mp2.size()); + EXPECT_TRUE(mp2.count(78731)); + EXPECT_TRUE(mp2.count(13141)); + EXPECT_EQ(99, mp2.at(78731)); + EXPECT_EQ(101, mp2.at(2651)); +} + +TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { + parallel_flat_hash_map mp1; + OutputArchiveWrapper w_out("./"); + + mp1[100] = 99; + mp1[300] = 299; + mp1[101] = 992; + mp1[1300] = 2991; + mp1[1130] = 299; + mp1[2130] = 1299; + + EXPECT_TRUE(mp1.dump(w_out)); + parallel_flat_hash_map mp2; + InputArchiveWrapper w_in("./"); + + EXPECT_TRUE(mp2.load(w_in)); + + EXPECT_EQ(6, mp2.size()); + EXPECT_EQ(99, mp2[100]); + EXPECT_EQ(299, mp2[300]); + EXPECT_EQ(299, mp2[1130]); + EXPECT_EQ(1299, mp2[2130]); +} + +} +} +} \ No newline at end of file From 21c032c9a5f61ffc95782330d3f9afc194c1c024 Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Sun, 18 Aug 2019 22:50:47 +0800 Subject: [PATCH 06/13] added tests --- examples/dump_load.cc | 2 +- tests/dump_load_test.cc | 36 +++++++----------------------------- 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index 8a77e6c..bec9e48 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -54,4 +54,4 @@ int main() dump_load_uint64_uint32(); dump_load_parallel_flat_hash_map(); return 0; -} \ No newline at end of file +} diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc index 8424a5d..b4dc2fe 100644 --- a/tests/dump_load_test.cc +++ b/tests/dump_load_test.cc @@ -16,43 +16,22 @@ using ::phmap::BinaryInputArchive; using ::phmap::OutputArchiveWrapper; using ::phmap::InputArchiveWrapper; -TEST(DumpLoad, FlatHashSet_string) { - flat_hash_set st1; +TEST(DumpLoad, FlatHashSet_uin32) { + flat_hash_set st1; BinaryOutputArchive ar_out("./dump.data"); - st1.insert("hello"); - st1.insert("world"); + st1.insert(1991); + st1.insert(1202); EXPECT_TRUE(st1.dump(ar_out)); - flat_hash_set st2; + flat_hash_set st2; BinaryInputArchive ar_in("./dump.data"); EXPECT_TRUE(st2.load(ar_in)); EXPECT_EQ(2, st2.size()); - EXPECT_TRUE(st2.count("hello")); - EXPECT_TRUE(st2.count("world")); -} - -TEST(DumpLoad, FlatHashMap_string_uint32) { - flat_hash_map mp1; - BinaryOutputArchive ar_out("./dump.data"); - - mp1["key-1"] = 99; - mp1["key-2"] = 299; - - EXPECT_TRUE(mp1.dump(ar_out)); - flat_hash_map mp2; - BinaryInputArchive ar_in("./dump.data"); - - EXPECT_TRUE(mp2.load(ar_in)); - - EXPECT_EQ(2, mp2.size()); - - EXPECT_TRUE(mp2.count("key-1")); - EXPECT_TRUE(mp2.count("key-2")); - EXPECT_EQ(99, mp2.at("key-1")); - EXPECT_EQ(299, mp2.at("key-2")); + EXPECT_TRUE(st2.count(1991)); + EXPECT_TRUE(st2.count(1202)); } TEST(DumpLoad, FlatHashMap_uint64_uint32) { @@ -102,4 +81,3 @@ TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { } } -} \ No newline at end of file From 6ae6e2ee495a0b052f63809e23f456f0ec889382 Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Tue, 20 Aug 2019 00:48:22 +0800 Subject: [PATCH 07/13] only support dump&load is_trivially_copyable types --- examples/dump_load.cc | 1 + parallel_hashmap/phmap.h | 72 +++++----------------------------- parallel_hashmap/phmap_base.h | 25 ++++-------- parallel_hashmap/phmap_utils.h | 43 ++++++-------------- tests/dump_load_test.cc | 5 ++- 5 files changed, 34 insertions(+), 112 deletions(-) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index bec9e48..4c92113 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -55,3 +55,4 @@ int main() dump_load_parallel_flat_hash_map(); return 0; } + diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index de42395..05e8b25 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1538,7 +1538,7 @@ public: } template - typename std::enable_if::value, bool>::type + typename std::enable_if::value, bool>::type dump(OutputArchive& ar) { typename OutputArchive::Guard guard(&ar); if (!ar.dump(size_)) { @@ -1547,7 +1547,7 @@ public: } if (size_ == 0) { return true; - } + } if (!ar.dump(capacity_)) { std::cerr << "Failed to dump capacity_" << std::endl; return false; @@ -1561,12 +1561,12 @@ public: if (!ar.dump(reinterpret_cast(slots_), sizeof(slot_type) * capacity_)) { std::cerr << "Failed to dump slot_" << std::endl; return false; - } + } return true; } template - typename std::enable_if::value, bool>::type + typename std::enable_if::value, bool>::type load(InputArchive& ar) { typename InputArchive::Guard guard(&ar); if (!ar.load(&size_)){ @@ -1594,60 +1594,6 @@ public: return true; } - // V will be V for hash_set and std::pair for hash_map - template - typename std::enable_if::value - && type_traits_internal::IsStringOrArithmeticType::value, bool>::type - dump(OutputArchive& ar) { - typename OutputArchive::Guard guard(&ar); - if (!ar.template dump(size_)) { - std::cerr << "Failed to dump size" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - for (auto it = this->begin(); it != this->end(); ++it) { - if (!ar.template dump(*it)) { - std::cerr << "Failed to dump element" << std::endl; - return false; - } - } - return true; - } - - template - typename std::enable_if::value - && type_traits_internal::IsStringOrArithmeticType::value, bool>::type - load(InputArchive& ar) { - typename InputArchive::Guard guard(&ar); - size_t sz = 0; - ar.template load(&sz); - for (size_t i = 0; i < sz; i ++) { - V v; - if (!ar.template load(&v)) { - std::cerr << "Failed to load element " << i << std::endl; - return false; - } - this->insert(v); - } - return true; - } - - template - typename std::enable_if::value, bool>::type - dump(OutputArchive&) { - std::cerr << "Does not support this type now!" << std::endl; - return false; - } - - template - typename std::enable_if::value, bool>::type - load(InputArchive&) { - std::cerr << "Does not support this type now!" << std::endl; - return false; - } - void rehash(size_t n) { if (n == 0 && capacity_ == 0) return; if (n == 0 && size_ == 0) { @@ -3253,8 +3199,9 @@ public: a.swap(b); } - template - bool dump(OutputArchiveWrapper& w) { + template + typename std::enable_if::value, bool>::type + dump(OutputArchiveWrapper& w) { for (size_t i = 0; i < sets_.size(); ++i) { auto& inner = sets_[i]; auto ar = w.create_archive(i); @@ -3272,8 +3219,9 @@ public: return true; } - template - bool load(InputArchiveWrapper& w) { + template + typename std::enable_if::value, bool>::type + load(InputArchiveWrapper& w) { size_t submap_count = w.load_meta(); if (submap_count != subcnt()) { diff --git a/parallel_hashmap/phmap_base.h b/parallel_hashmap/phmap_base.h index 8f03623..82a38b0 100644 --- a/parallel_hashmap/phmap_base.h +++ b/parallel_hashmap/phmap_base.h @@ -78,27 +78,16 @@ struct PairTrait : public std::false_type { template struct PairTrait>: public std::true_type { - using first_type = typename std::remove_cv::type; - using second_type = typename std::remove_cv::type; + using first_type = T1; + using second_type = T2; }; template -struct IsArithmeticType { - static constexpr bool value = std::is_arithmetic::value - || (PairTrait::value && - std::is_arithmetic::first_type>::value - && std::is_arithmetic::second_type>::value); -}; - -template -struct IsStringOrArithmeticType { - static constexpr bool value = IsArithmeticType::value - || std::is_same::value - || (PairTrait::value - && (std::is_arithmetic::first_type>::value - || std::is_same::first_type, std::string>::value) - && (std::is_arithmetic::second_type>::value - || std::is_same::second_type, std::string>::value)); +struct IsDumpableType { + static constexpr bool value = std::is_trivially_copyable::value + || (PairTrait::value + && std::is_trivially_copyable::first_type>::value + && std::is_trivially_copyable::second_type>::value); }; template diff --git a/parallel_hashmap/phmap_utils.h b/parallel_hashmap/phmap_utils.h index 3807498..e517afd 100644 --- a/parallel_hashmap/phmap_utils.h +++ b/parallel_hashmap/phmap_utils.h @@ -351,29 +351,21 @@ public: } template - typename std::enable_if::value, bool>::type + typename std::enable_if::value, bool>::type dump(const V& v) { CHECK_FILE(ofs_); ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); return true; } - template - typename std::enable_if::type>::value, bool>::type - dump(const V& v) { - CHECK_FILE(ofs_); - uint32_t sz = v.length(); - ofs_.write(reinterpret_cast(&sz), sizeof(sz)); - ofs_.write(const_cast(v.data()), sz); - return true; - } - template typename std::enable_if::value - && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + && type_traits_internal::IsDumpableType::value, bool>::type dump(const V& v) { - return dump::first_type>(v.first) - && dump::second_type>(v.second); + using first_type = typename type_traits_internal::PairTrait::first_type; + using second_type = typename type_traits_internal::PairTrait::second_type; + return dump(v.first) + && dump(v.second); } void finish() { @@ -405,32 +397,21 @@ public: } template - typename std::enable_if::value, bool>::type + typename std::enable_if::value, bool>::type load(V* v) { CHECK_FILE(ifs_); ifs_.read(reinterpret_cast(v), sizeof(V)); return true; } - template - typename std::enable_if::type>::value, bool>::type - load(V* v) { - CHECK_FILE(ifs_); - uint32_t sz = 0; - ifs_.read(reinterpret_cast(&sz), sizeof(sz)); - const_cast(v)->resize(sz); - ifs_.read(const_cast(v->data()), sz); - return true; - } - template typename std::enable_if::value - && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + && type_traits_internal::IsDumpableType::value, bool>::type load(V* v) { - using first_type = typename type_traits_internal::PairTrait::first_type; - using second_type = typename type_traits_internal::PairTrait::second_type; - return load(const_cast(&v->first)) - && load(const_cast(&v->second)); + using first_type = typename std::remove_cv::first_type>::type; + using second_type = typename std::remove_cv::second_type>::type; + return load(const_cast(&v->first)) + && load(const_cast(&v->second)); } void finish() { diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc index b4dc2fe..e162ba0 100644 --- a/tests/dump_load_test.cc +++ b/tests/dump_load_test.cc @@ -8,6 +8,7 @@ namespace phmap { namespace container_internal { namespace { + using ::phmap::flat_hash_set; using ::phmap::flat_hash_map; using ::phmap::parallel_flat_hash_map; @@ -22,7 +23,7 @@ TEST(DumpLoad, FlatHashSet_uin32) { st1.insert(1991); st1.insert(1202); - + EXPECT_TRUE(st1.dump(ar_out)); flat_hash_set st2; BinaryInputArchive ar_in("./dump.data"); @@ -81,3 +82,5 @@ TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { } } +} + From e3c55f983099d0a614449ec641adfd224fe38ad6 Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Tue, 20 Aug 2019 09:54:09 +0800 Subject: [PATCH 08/13] support older version cpp compiler to use is_trivially_copyable --- parallel_hashmap/phmap_base.h | 15 ++++++++++++--- parallel_hashmap/phmap_utils.h | 26 +++----------------------- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/parallel_hashmap/phmap_base.h b/parallel_hashmap/phmap_base.h index 82a38b0..b2534bd 100644 --- a/parallel_hashmap/phmap_base.h +++ b/parallel_hashmap/phmap_base.h @@ -82,12 +82,21 @@ struct PairTrait>: public std::true_type { using second_type = T2; }; +template +#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801 +struct IsTriviallyCopyable : public std::integral_constant { +}; +#else +struct IsTriviallyCopyable : public std::is_trivially_copyable { +}; +#endif + template struct IsDumpableType { - static constexpr bool value = std::is_trivially_copyable::value + static constexpr bool value = IsTriviallyCopyable::value || (PairTrait::value - && std::is_trivially_copyable::first_type>::value - && std::is_trivially_copyable::second_type>::value); + && IsTriviallyCopyable::first_type>::value + && IsTriviallyCopyable::second_type>::value); }; template diff --git a/parallel_hashmap/phmap_utils.h b/parallel_hashmap/phmap_utils.h index e517afd..9146057 100644 --- a/parallel_hashmap/phmap_utils.h +++ b/parallel_hashmap/phmap_utils.h @@ -326,7 +326,7 @@ public: ~ArchiveGuard() { if (ar_) { ar_->finish(); - } + } } private: Archive* ar_; @@ -351,23 +351,13 @@ public: } template - typename std::enable_if::value, bool>::type + typename std::enable_if::value, bool>::type dump(const V& v) { CHECK_FILE(ofs_); ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); return true; } - template - typename std::enable_if::value - && type_traits_internal::IsDumpableType::value, bool>::type - dump(const V& v) { - using first_type = typename type_traits_internal::PairTrait::first_type; - using second_type = typename type_traits_internal::PairTrait::second_type; - return dump(v.first) - && dump(v.second); - } - void finish() { if (ofs_.is_open()) { ofs_.close(); @@ -397,23 +387,13 @@ public: } template - typename std::enable_if::value, bool>::type + typename std::enable_if::value, bool>::type load(V* v) { CHECK_FILE(ifs_); ifs_.read(reinterpret_cast(v), sizeof(V)); return true; } - template - typename std::enable_if::value - && type_traits_internal::IsDumpableType::value, bool>::type - load(V* v) { - using first_type = typename std::remove_cv::first_type>::type; - using second_type = typename std::remove_cv::second_type>::type; - return load(const_cast(&v->first)) - && load(const_cast(&v->second)); - } - void finish() { if (ifs_.is_open()) { ifs_.close(); From 0866192922a8b8f8bd95d97fa411365d5fa66988 Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Wed, 21 Aug 2019 09:01:56 +0800 Subject: [PATCH 09/13] use single file to store data for parallel hash --- examples/dump_load.cc | 8 ++-- parallel_hashmap/phmap.h | 34 +++++++++-------- parallel_hashmap/phmap_utils.h | 70 ++++++---------------------------- tests/dump_load_test.cc | 10 ++--- 4 files changed, 39 insertions(+), 83 deletions(-) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index 4c92113..6369c0c 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -27,7 +27,7 @@ void dump_load_uint64_uint32() { void dump_load_parallel_flat_hash_map() { parallel_flat_hash_map mp1; - phmap::OutputArchiveWrapper w_out("./"); + phmap::BinaryOutputArchive ar_out("./dump.data"); // Add a new entry mp1[100] = 99; @@ -40,11 +40,11 @@ void dump_load_parallel_flat_hash_map() { for (const auto& n : mp1) std::cout << "key: " << n.first << ", value: " << n.second << "\n"; - mp1.dump(w_out); + mp1.dump(ar_out); parallel_flat_hash_map mp2; - phmap::InputArchiveWrapper w_in("./"); + phmap::BinaryInputArchive ar_in("./dump.data"); - mp2.load(w_in); + mp2.load(ar_in); for (const auto& n : mp2) std::cout << "key: " << n.first << ", value: " << n.second << "\n"; } diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 05e8b25..506e941 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -3199,40 +3199,44 @@ public: a.swap(b); } - template + template typename std::enable_if::value, bool>::type - dump(OutputArchiveWrapper& w) { + dump(OutputArchive& ar) { + typename OutputArchive::Guard guard(&ar); + if (! ar.dump(subcnt())) { + std::cerr << "Failed to dump meta!" << std::endl; + return false; + } for (size_t i = 0; i < sets_.size(); ++i) { auto& inner = sets_[i]; - auto ar = w.create_archive(i); typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.dump(*ar)) { + if (!inner.set_.dump(ar)) { std::cerr << "Failed to dump submap " << i << std::endl; return false; } } - - if (! w.dump_meta(subcnt())) { - std::cerr << "Failed to dump meta!" << std::endl; - return false; - } return true; } - template + template typename std::enable_if::value, bool>::type - load(InputArchiveWrapper& w) { - size_t submap_count = w.load_meta(); + load(InputArchive& ar) { + typename InputArchive::Guard guard(&ar); + size_t submap_count = 0; + if (!ar.load(&submap_count)) { + std::cerr << "Failed to load submap count!" << std::endl; + return false; + } if (submap_count != subcnt()) { std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; return false; } - for (size_t i = 0; i < sets_.size(); ++i) { - auto ar = w.create_archive(i); + for (size_t i = 0; i < submap_count; ++i) { auto& inner = sets_[i]; - if (!inner.set_.load(*ar)) { + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.load(ar)) { std::cerr << "Failed to load submap " << i << std::endl; return false; } diff --git a/parallel_hashmap/phmap_utils.h b/parallel_hashmap/phmap_utils.h index 9146057..f9c34f7 100644 --- a/parallel_hashmap/phmap_utils.h +++ b/parallel_hashmap/phmap_utils.h @@ -322,9 +322,13 @@ using HashState = HashStateBase; template class ArchiveGuard { public: - ArchiveGuard(Archive* ar): ar_(ar) {}; + ArchiveGuard(Archive* ar): ar_(ar) { + if (ar_->guard_ == NULL) { + ar_->guard_ = this; + } + }; ~ArchiveGuard() { - if (ar_) { + if (ar_ && ar_->guard_ == this) { ar_->finish(); } } @@ -336,7 +340,7 @@ class BinaryOutputArchive { public: using Guard = ArchiveGuard; - BinaryOutputArchive(const std::string& file_path) { + BinaryOutputArchive(const std::string& file_path): guard_(NULL) { ofs_.open(file_path.c_str(), std::ios_base::binary); } @@ -364,7 +368,9 @@ public: } } private: + friend class ArchiveGuard; std::ofstream ofs_; + Guard* guard_; }; @@ -372,7 +378,7 @@ class BinaryInputArchive { public: using Guard = ArchiveGuard; - BinaryInputArchive(const std::string& file_path) { + BinaryInputArchive(const std::string& file_path): guard_(NULL) { ifs_.open(file_path.c_str(), std::ios_base::binary); } @@ -400,62 +406,10 @@ public: } } private: + friend class ArchiveGuard; std::ifstream ifs_; + Guard* guard_; }; - -template -class OutputArchiveWrapper { -public: - using SubArchive = T; - OutputArchiveWrapper(const std::string& dir): dir_(dir) { - } - - virtual ~OutputArchiveWrapper() { - - } - - bool dump_meta(size_t subcnt) { - auto ar = std::make_shared(dir_ + "/meta.dump"); - typename SubArchive::Guard guard(ar.get()); - ar->dump(subcnt); - return true; - } - - std::shared_ptr create_archive(size_t i) { - std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump"; - return std::make_shared(file_path); - } -private: - std::string dir_; -}; - -template -class InputArchiveWrapper { -public: - using SubArchive = T; - InputArchiveWrapper(const std::string& dir): dir_(dir) { - } - - virtual ~InputArchiveWrapper() { - - } - - size_t load_meta() { - size_t subcnt = 0; - auto ar = std::make_shared(dir_ + "/meta.dump"); - typename SubArchive::Guard guard(ar.get()); - ar->load(&subcnt); - return subcnt; - } - - std::shared_ptr create_archive(size_t i) { - std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump"; - return std::make_shared(file_path); - } -private: - std::string dir_; -}; - } // namespace phmap diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc index e162ba0..e557771 100644 --- a/tests/dump_load_test.cc +++ b/tests/dump_load_test.cc @@ -14,8 +14,6 @@ using ::phmap::flat_hash_map; using ::phmap::parallel_flat_hash_map; using ::phmap::BinaryOutputArchive; using ::phmap::BinaryInputArchive; -using ::phmap::OutputArchiveWrapper; -using ::phmap::InputArchiveWrapper; TEST(DumpLoad, FlatHashSet_uin32) { flat_hash_set st1; @@ -58,7 +56,7 @@ TEST(DumpLoad, FlatHashMap_uint64_uint32) { TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { parallel_flat_hash_map mp1; - OutputArchiveWrapper w_out("./"); + BinaryOutputArchive ar_out("./dump.data"); mp1[100] = 99; mp1[300] = 299; @@ -67,11 +65,11 @@ TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { mp1[1130] = 299; mp1[2130] = 1299; - EXPECT_TRUE(mp1.dump(w_out)); + EXPECT_TRUE(mp1.dump(ar_out)); parallel_flat_hash_map mp2; - InputArchiveWrapper w_in("./"); + BinaryInputArchive ar_in("./dump.data"); - EXPECT_TRUE(mp2.load(w_in)); + EXPECT_TRUE(mp2.load(ar_in)); EXPECT_EQ(6, mp2.size()); EXPECT_EQ(99, mp2[100]); From 61f1bdb037b336ebf6225fec1d4b193031a766c9 Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Wed, 21 Aug 2019 09:01:56 +0800 Subject: [PATCH 10/13] use single file to store data for parallel hash --- examples/dump_load.cc | 8 ++-- parallel_hashmap/phmap.h | 38 +++++++++--------- parallel_hashmap/phmap_utils.h | 70 ++++++---------------------------- tests/dump_load_test.cc | 10 ++--- 4 files changed, 41 insertions(+), 85 deletions(-) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index 4c92113..6369c0c 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -27,7 +27,7 @@ void dump_load_uint64_uint32() { void dump_load_parallel_flat_hash_map() { parallel_flat_hash_map mp1; - phmap::OutputArchiveWrapper w_out("./"); + phmap::BinaryOutputArchive ar_out("./dump.data"); // Add a new entry mp1[100] = 99; @@ -40,11 +40,11 @@ void dump_load_parallel_flat_hash_map() { for (const auto& n : mp1) std::cout << "key: " << n.first << ", value: " << n.second << "\n"; - mp1.dump(w_out); + mp1.dump(ar_out); parallel_flat_hash_map mp2; - phmap::InputArchiveWrapper w_in("./"); + phmap::BinaryInputArchive ar_in("./dump.data"); - mp2.load(w_in); + mp2.load(ar_in); for (const auto& n : mp2) std::cout << "key: " << n.first << ", value: " << n.second << "\n"; } diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 05e8b25..69c3baf 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1553,7 +1553,7 @@ public: return false; } if (!ar.dump(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { std::cerr << "Failed to dump ctrl_" << std::endl; return false; @@ -1583,7 +1583,7 @@ public: // allocate memory for ctrl_ and slots_ initialize_slots(); if (!ar.load(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { std::cerr << "Failed to load ctrl" << std::endl; return false; } @@ -3199,40 +3199,44 @@ public: a.swap(b); } - template + template typename std::enable_if::value, bool>::type - dump(OutputArchiveWrapper& w) { + dump(OutputArchive& ar) { + typename OutputArchive::Guard guard(&ar); + if (! ar.dump(subcnt())) { + std::cerr << "Failed to dump meta!" << std::endl; + return false; + } for (size_t i = 0; i < sets_.size(); ++i) { auto& inner = sets_[i]; - auto ar = w.create_archive(i); typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.dump(*ar)) { + if (!inner.set_.dump(ar)) { std::cerr << "Failed to dump submap " << i << std::endl; return false; } } - - if (! w.dump_meta(subcnt())) { - std::cerr << "Failed to dump meta!" << std::endl; - return false; - } return true; } - template + template typename std::enable_if::value, bool>::type - load(InputArchiveWrapper& w) { - size_t submap_count = w.load_meta(); + load(InputArchive& ar) { + typename InputArchive::Guard guard(&ar); + size_t submap_count = 0; + if (!ar.load(&submap_count)) { + std::cerr << "Failed to load submap count!" << std::endl; + return false; + } if (submap_count != subcnt()) { std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; return false; } - for (size_t i = 0; i < sets_.size(); ++i) { - auto ar = w.create_archive(i); + for (size_t i = 0; i < submap_count; ++i) { auto& inner = sets_[i]; - if (!inner.set_.load(*ar)) { + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.load(ar)) { std::cerr << "Failed to load submap " << i << std::endl; return false; } diff --git a/parallel_hashmap/phmap_utils.h b/parallel_hashmap/phmap_utils.h index 9146057..f9c34f7 100644 --- a/parallel_hashmap/phmap_utils.h +++ b/parallel_hashmap/phmap_utils.h @@ -322,9 +322,13 @@ using HashState = HashStateBase; template class ArchiveGuard { public: - ArchiveGuard(Archive* ar): ar_(ar) {}; + ArchiveGuard(Archive* ar): ar_(ar) { + if (ar_->guard_ == NULL) { + ar_->guard_ = this; + } + }; ~ArchiveGuard() { - if (ar_) { + if (ar_ && ar_->guard_ == this) { ar_->finish(); } } @@ -336,7 +340,7 @@ class BinaryOutputArchive { public: using Guard = ArchiveGuard; - BinaryOutputArchive(const std::string& file_path) { + BinaryOutputArchive(const std::string& file_path): guard_(NULL) { ofs_.open(file_path.c_str(), std::ios_base::binary); } @@ -364,7 +368,9 @@ public: } } private: + friend class ArchiveGuard; std::ofstream ofs_; + Guard* guard_; }; @@ -372,7 +378,7 @@ class BinaryInputArchive { public: using Guard = ArchiveGuard; - BinaryInputArchive(const std::string& file_path) { + BinaryInputArchive(const std::string& file_path): guard_(NULL) { ifs_.open(file_path.c_str(), std::ios_base::binary); } @@ -400,62 +406,10 @@ public: } } private: + friend class ArchiveGuard; std::ifstream ifs_; + Guard* guard_; }; - -template -class OutputArchiveWrapper { -public: - using SubArchive = T; - OutputArchiveWrapper(const std::string& dir): dir_(dir) { - } - - virtual ~OutputArchiveWrapper() { - - } - - bool dump_meta(size_t subcnt) { - auto ar = std::make_shared(dir_ + "/meta.dump"); - typename SubArchive::Guard guard(ar.get()); - ar->dump(subcnt); - return true; - } - - std::shared_ptr create_archive(size_t i) { - std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump"; - return std::make_shared(file_path); - } -private: - std::string dir_; -}; - -template -class InputArchiveWrapper { -public: - using SubArchive = T; - InputArchiveWrapper(const std::string& dir): dir_(dir) { - } - - virtual ~InputArchiveWrapper() { - - } - - size_t load_meta() { - size_t subcnt = 0; - auto ar = std::make_shared(dir_ + "/meta.dump"); - typename SubArchive::Guard guard(ar.get()); - ar->load(&subcnt); - return subcnt; - } - - std::shared_ptr create_archive(size_t i) { - std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump"; - return std::make_shared(file_path); - } -private: - std::string dir_; -}; - } // namespace phmap diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc index e162ba0..e557771 100644 --- a/tests/dump_load_test.cc +++ b/tests/dump_load_test.cc @@ -14,8 +14,6 @@ using ::phmap::flat_hash_map; using ::phmap::parallel_flat_hash_map; using ::phmap::BinaryOutputArchive; using ::phmap::BinaryInputArchive; -using ::phmap::OutputArchiveWrapper; -using ::phmap::InputArchiveWrapper; TEST(DumpLoad, FlatHashSet_uin32) { flat_hash_set st1; @@ -58,7 +56,7 @@ TEST(DumpLoad, FlatHashMap_uint64_uint32) { TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { parallel_flat_hash_map mp1; - OutputArchiveWrapper w_out("./"); + BinaryOutputArchive ar_out("./dump.data"); mp1[100] = 99; mp1[300] = 299; @@ -67,11 +65,11 @@ TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { mp1[1130] = 299; mp1[2130] = 1299; - EXPECT_TRUE(mp1.dump(w_out)); + EXPECT_TRUE(mp1.dump(ar_out)); parallel_flat_hash_map mp2; - InputArchiveWrapper w_in("./"); + BinaryInputArchive ar_in("./dump.data"); - EXPECT_TRUE(mp2.load(w_in)); + EXPECT_TRUE(mp2.load(ar_in)); EXPECT_EQ(6, mp2.size()); EXPECT_EQ(99, mp2[100]); From 34ecd92f63ec0145578bc1c7c6adc222310638e5 Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Fri, 23 Aug 2019 10:21:13 +0800 Subject: [PATCH 11/13] added mmap load --- examples/dump_load.cc | 77 +++- parallel_hashmap/phmap.h | 115 +----- parallel_hashmap/phmap_base.h | 30 +- parallel_hashmap/phmap_dump.h | 635 +++++++++++++++++++++++++++++++++ parallel_hashmap/phmap_utils.h | 106 +----- tests/dump_load_test.cc | 68 +++- 6 files changed, 795 insertions(+), 236 deletions(-) create mode 100644 parallel_hashmap/phmap_dump.h diff --git a/examples/dump_load.cc b/examples/dump_load.cc index 6369c0c..ec8cba7 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -1,6 +1,6 @@ #include #include -#include +#include using phmap::flat_hash_map; using phmap::parallel_flat_hash_map; @@ -21,7 +21,7 @@ void dump_load_uint64_uint32() { phmap::BinaryInputArchive ar_in("./dump.data"); mp2.load(ar_in); // Iterate and print keys and values g|++ - for (const auto& n : mp2) + for (const auto& n : mp2) std::cout << n.first << "'s value is: " << n.second << "\n"; } @@ -49,10 +49,83 @@ void dump_load_parallel_flat_hash_map() { std::cout << "key: " << n.first << ", value: " << n.second << "\n"; } +#if defined(__linux__) +void mmap_load_uint64_uint32() { + using MapType = flat_hash_map, + phmap::container_internal::hash_default_eq, + phmap::MmapAllocator< + phmap::container_internal::Pair>>; + MapType mp1; + mp1.reserve(100); + phmap::MmapOutputArchive ar_out("./dump.data"); + // Add a new entry + mp1[100] = 99; + mp1[300] = 299; + + // Iterate and print keys and values + for (const auto& n : mp1) + std::cout << n.first << "'s value is: " << n.second << "\n"; + + mp1.mmap_dump(ar_out); + MapType mp2; + + phmap::MmapInputArchive ar_in("./dump.data"); + mp2.mmap_load(ar_in); + mp2[849242] = 141; + mp2[11] = 1111; + // Iterate and print keys and values g|++ + for (const auto& n : mp2) + std::cout << n.first << "'s value is: " << n.second << "\n"; +} + +void mmap_load_parallel_flat_hash_map() { + using MapType = parallel_flat_hash_map, + phmap::container_internal::hash_default_eq, + phmap::MmapAllocator< + phmap::container_internal::Pair>, + 4, + phmap::NullMutex>; + + MapType mp1; + phmap::MmapOutputArchive ar_out("./dump.data"); + + // Add a new entry + mp1[100] = 99; + mp1[300] = 299; + mp1[101] = 992; + mp1[1300] = 2991; + mp1[1130] = 299; + mp1[2130] = 1299; + // Iterate and print + for (const auto& n : mp1) + std::cout << "key: " << n.first << ", value: " << n.second << "\n"; + + mp1.mmap_dump(ar_out); + MapType mp2; + phmap::MmapInputArchive ar_in("./dump.data"); + + mp2.mmap_load(ar_in); + std::cout << "[debug] map capacity: " << mp2.capacity() << ", size: " << mp2.size() << std::endl; + for (size_t i = 0; i < 100; i ++) { + mp2[6771 + i] = i; + } + std::cout << "[debug] map capacity: " << mp2.capacity() << ", size: " << mp2.size() << std::endl; + for (const auto& n : mp2) + std::cout << "key: " << n.first << ", value: " << n.second << "\n"; +} +#endif + int main() { dump_load_uint64_uint32(); dump_load_parallel_flat_hash_map(); + +#if defined(__linux__) + mmap_load_uint64_uint32(); + mmap_load_parallel_flat_hash_map(); +#endif return 0; } diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 506e941..454a209 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -45,9 +45,6 @@ #include #include #include -#include -#include -#include #include "phmap_utils.h" #include "phmap_base.h" @@ -1537,62 +1534,17 @@ public: } } - template - typename std::enable_if::value, bool>::type - dump(OutputArchive& ar) { - typename OutputArchive::Guard guard(&ar); - if (!ar.dump(size_)) { - std::cerr << "Failed to dump size_" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.dump(capacity_)) { - std::cerr << "Failed to dump capacity_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { + template + bool dump(OutputArchive&); - std::cerr << "Failed to dump ctrl_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast(slots_), sizeof(slot_type) * capacity_)) { - std::cerr << "Failed to dump slot_" << std::endl; - return false; - } - return true; - } + template + bool load(InputArchive&); - template - typename std::enable_if::value, bool>::type - load(InputArchive& ar) { - typename InputArchive::Guard guard(&ar); - if (!ar.load(&size_)){ - std::cerr << "Failed to load size_" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.load(&capacity_)) { - std::cerr << "Failed to load capacity_" << std::endl; - return false; - } - // allocate memory for ctrl_ and slots_ - initialize_slots(); - if (!ar.load(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) { - std::cerr << "Failed to load ctrl" << std::endl; - return false; - } - if (!ar.load(reinterpret_cast(slots_), sizeof(slot_type) * capacity_)) { - std::cerr << "Failed to load slot" << std::endl; - return false; - } - return true; - } + template + bool mmap_dump(OutputArchive&); + + template + bool mmap_load(MmapInputArchive&); void rehash(size_t n) { if (n == 0 && capacity_ == 0) return; @@ -3199,50 +3151,17 @@ public: a.swap(b); } - template - typename std::enable_if::value, bool>::type - dump(OutputArchive& ar) { - typename OutputArchive::Guard guard(&ar); - if (! ar.dump(subcnt())) { - std::cerr << "Failed to dump meta!" << std::endl; - return false; - } - for (size_t i = 0; i < sets_.size(); ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.dump(ar)) { - std::cerr << "Failed to dump submap " << i << std::endl; - return false; - } - } - return true; - } + template + bool dump(OutputArchive& ar); - template - typename std::enable_if::value, bool>::type - load(InputArchive& ar) { - typename InputArchive::Guard guard(&ar); - size_t submap_count = 0; - if (!ar.load(&submap_count)) { - std::cerr << "Failed to load submap count!" << std::endl; - return false; - } + template + bool load(InputArchive& ar); - if (submap_count != subcnt()) { - std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; - return false; - } + template + bool mmap_dump(OutputArchive& ar); - for (size_t i = 0; i < submap_count; ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.load(ar)) { - std::cerr << "Failed to load submap " << i << std::endl; - return false; - } - } - return true; - } + template + bool mmap_load(InputArchive& ar); private: template diff --git a/parallel_hashmap/phmap_base.h b/parallel_hashmap/phmap_base.h index b2534bd..bbc6712 100644 --- a/parallel_hashmap/phmap_base.h +++ b/parallel_hashmap/phmap_base.h @@ -33,6 +33,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // --------------------------------------------------------------------------- + #include #include #include @@ -70,35 +71,6 @@ struct EqualTo namespace type_traits_internal { -template -struct PairTrait : public std::false_type { - using first_type = typename std::remove_cv::type; - using second_type = typename std::remove_cv::type; -}; - -template -struct PairTrait>: public std::true_type { - using first_type = T1; - using second_type = T2; -}; - -template -#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801 -struct IsTriviallyCopyable : public std::integral_constant { -}; -#else -struct IsTriviallyCopyable : public std::is_trivially_copyable { -}; -#endif - -template -struct IsDumpableType { - static constexpr bool value = IsTriviallyCopyable::value - || (PairTrait::value - && IsTriviallyCopyable::first_type>::value - && IsTriviallyCopyable::second_type>::value); -}; - template struct VoidTImpl { using type = void; diff --git a/parallel_hashmap/phmap_dump.h b/parallel_hashmap/phmap_dump.h new file mode 100644 index 0000000..b75262a --- /dev/null +++ b/parallel_hashmap/phmap_dump.h @@ -0,0 +1,635 @@ +#if !defined(phmap_dump_h_guard_) +#define phmap_dump_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com +// +// providing dump/load/mmap_load +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +#include +#include +#include +#include +#include +#include "phmap.h" +namespace phmap +{ + +namespace type_traits_internal { + +#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801 + template struct IsTriviallyCopyable : public std::integral_constant {}; +#else + template struct IsTriviallyCopyable : public std::is_trivially_copyable {}; +#endif + +template +struct IsTriviallyCopyable> { + static constexpr bool value = IsTriviallyCopyable::value && IsTriviallyCopyable::value; +}; +} + +namespace container_internal { + +//// raw_hash_set +template +template +bool raw_hash_set::dump(OutputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename OutputArchive::Guard guard(&ar); + if (!ar.dump(size_)) { + std::cerr << "Failed to dump size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.dump(capacity_)) { + std::cerr << "Failed to dump capacity_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { + + std::cerr << "Failed to dump ctrl_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(slots_), + sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to dump slot_" << std::endl; + return false; + } + return true; +} + +template +template +bool raw_hash_set::load(InputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename InputArchive::Guard guard(&ar); + if (!ar.load(&size_)) { + std::cerr << "Failed to load size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.load(&capacity_)) { + std::cerr << "Failed to load capacity_" << std::endl; + return false; + } + + // allocate memory for ctrl_ and slots_ + initialize_slots(); + if (!ar.load(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { + std::cerr << "Failed to load ctrl" << std::endl; + return false; + } + if (!ar.load(reinterpret_cast(slots_), + sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to load slot" << std::endl; + return false; + } + return true; +} + + +template +template +bool raw_hash_set::mmap_dump(OutputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename OutputArchive::Guard guard(&ar); + size_t align_size = Layout::Alignment(); + if (!ar.dump(size_, align_size)) { + std::cerr << "Failed to dump size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.dump(capacity_, align_size)) { + std::cerr << "Failed to dump capacity_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1), align_size)) { + + std::cerr << "Failed to dump ctrl_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(slots_), + sizeof(slot_type) * capacity_, align_size)) { + std::cerr << "Failed to dump slot_" << std::endl; + return false; + } + return true; +} + +template +template +bool raw_hash_set::mmap_load(MmapInputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + assert(ar.initialized()); + auto closure = ar.closure(); + this->alloc_ref().set_closure(closure); + size_t align_size = Layout::Alignment(); + if (!ar.load(&size_, align_size)) { + std::cerr << "Failed to load size!" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.load(&capacity_, align_size)) { + std::cerr << "Failed to load capacity!" << std::endl; + return false; + } + + + if (std::is_same>::value) { + infoz_ = Sample(); + } + reset_growth_left(); + infoz_.RecordStorageChanged(size_, capacity_); + + char* p_ctrl = ar.load( + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1), align_size); + ctrl_ = reinterpret_cast(p_ctrl); + + char* p_slots = ar.load(sizeof(slot_type) * capacity_, align_size); + slots_ = reinterpret_cast(p_slots); + return true; +} + + +////// parallel_hash_set +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template +bool parallel_hash_set::dump(OutputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename OutputArchive::Guard guard(&ar); + if (! ar.dump(subcnt())) { + std::cerr << "Failed to dump meta!" << std::endl; + return false; + } + for (size_t i = 0; i < sets_.size(); ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.dump(ar)) { + std::cerr << "Failed to dump submap " << i << std::endl; + return false; + } + } + return true; +} + +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template +bool parallel_hash_set::load(InputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename InputArchive::Guard guard(&ar); + size_t submap_count = 0; + if (!ar.load(&submap_count)) { + std::cerr << "Failed to load submap count!" << std::endl; + return false; + } + + if (submap_count != subcnt()) { + std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; + return false; + } + + for (size_t i = 0; i < submap_count; ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.load(ar)) { + std::cerr << "Failed to load submap " << i << std::endl; + return false; + } + } + return true; +} + +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template +bool parallel_hash_set::mmap_dump(OutputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename OutputArchive::Guard guard(&ar); + size_t align_size = EmbeddedSet::Layout::Alignment(); + if (! ar.dump(subcnt(), align_size)) { + std::cerr << "Failed to dump meta!" << std::endl; + return false; + } + for (size_t i = 0; i < sets_.size(); ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.mmap_dump(ar)) { + std::cerr << "Failed to dump submap " << i << std::endl; + return false; + } + } + return true; +} + +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template +bool parallel_hash_set::mmap_load(InputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + assert(ar.initialized()); + auto closure = ar.closure(); + this->alloc_ref().set_closure(closure); + size_t submap_count = 0; + size_t align_size = EmbeddedSet::Layout::Alignment(); + if (!ar.load(&submap_count, align_size)) { + std::cerr << "Failed to load submap count!" << std::endl; + return false; + } + + if (submap_count != subcnt()) { + std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; + return false; + } + + for (size_t i = 0; i < submap_count; ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.mmap_load(ar)) { + std::cerr << "Failed to load submap " << i << std::endl; + return false; + } + } + return true; +} +} // namesapce container_internal + + + +// ArchiveOutput & ArchiveInput + +#define CHECK_FILE(f) { \ + if (!f.is_open()) { \ + std::cerr << "File is not open!" << std::endl; \ + return false; \ + } \ +} + +template +class ArchiveGuard { +public: + ArchiveGuard(Archive* ar): ar_(ar) { + if (ar_->guard_ == NULL) { + ar_->guard_ = this; + } + }; + ~ArchiveGuard() { + if (ar_ && ar_->guard_ == this) { + ar_->finish(); + } + } +private: + Archive* ar_; +}; + +class BinaryOutputArchive { +public: + using Guard = ArchiveGuard; + + BinaryOutputArchive(const std::string& file_path): offset_(0), guard_(NULL) { + ofs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~BinaryOutputArchive() { + finish(); + } + + bool dump(char* p, size_t sz) { + CHECK_FILE(ofs_); + ofs_.write(p, sz); + offset_ += sz; + return true; + } + + template + typename std::enable_if::value, bool>::type + dump(const V& v) { + CHECK_FILE(ofs_); + ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); + offset_ += sizeof(V); + return true; + } + + void finish() { + if (ofs_.is_open()) { + ofs_.close(); + offset_ = 0; + } + } + +private: + friend class ArchiveGuard; + std::ofstream ofs_; + size_t offset_; + Guard* guard_; +}; + + +class BinaryInputArchive { +public: + using Guard = ArchiveGuard; + + BinaryInputArchive(const std::string& file_path): offset_(0), guard_(NULL) { + ifs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~BinaryInputArchive() { + finish(); + } + + bool load(char* p, size_t sz) { + CHECK_FILE(ifs_); + ifs_.read(p, sz); + offset_ += sz; + return true; + } + + template + typename std::enable_if::value, bool>::type + load(V* v) { + CHECK_FILE(ifs_); + ifs_.read(reinterpret_cast(v), sizeof(V)); + offset_ += sizeof(V); + return true; + } + + void finish() { + if (ifs_.is_open()) { + ifs_.close(); + offset_ = 0; + } + } + +private: + friend class ArchiveGuard; + std::ifstream ifs_; + size_t offset_; + Guard* guard_; +}; + +#if defined(__linux__) // only support linux's mmap now +// mmap dump && mmap load +#include +#include +#include +#include + +class MmapOutputArchive { +public: + using Guard = ArchiveGuard; + + MmapOutputArchive(const std::string& file_path): offset_(0), guard_(NULL) { + ofs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~MmapOutputArchive() { + finish(); + } + + bool dump(char* p, size_t sz, size_t align_size) { + CHECK_FILE(ofs_); + ofs_.write(p, sz); + offset_ += sz; + return align(align_size); + } + + template + typename std::enable_if::value, bool>::type + dump(const V& v, size_t align_size) { + CHECK_FILE(ofs_); + ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); + offset_ += sizeof(V); + return align(align_size); + } + + void finish() { + if (ofs_.is_open()) { + ofs_.close(); + offset_ = 0; + } + } +private: + // padding for align + bool align(size_t align) { + size_t padding_size = (offset_ + align - 1) / align * align - offset_; + if (padding_size == 0) { + return true; + } + std::string padding(padding_size, '\0'); + ofs_.write(padding.c_str(), padding_size); + offset_ += padding_size; + return true; + } + + friend class ArchiveGuard; + std::ofstream ofs_; + size_t offset_; + Guard* guard_; +}; + + +class MmapInputArchive { +public: + class MmapClosure { + public: + MmapClosure(bool init = false, size_t len = 0, void* a = NULL): + initialized(init), length(len), addr(a) {}; + + ~MmapClosure() { + if (initialized && addr != MAP_FAILED) { + munmap(addr, length); + } + } + bool initialized; + size_t length; + void* addr; + }; + + MmapInputArchive(const std::string& file_path): + initialized_(false), closure_(nullptr) { + int fd = open(file_path.c_str(), O_RDONLY); + if (fd == -1) { + std::cerr << "Failed to open file " << file_path << std::endl; + return; + } + + struct stat st; + if (fstat(fd, &st) == -1) { + std::cerr << "Failed to stat file " << file_path << std::endl; + close(fd); + return; + } + + file_size_ = st.st_size; + + addr_ = mmap(NULL, file_size_, PROT_READ|PROT_WRITE, + MAP_PRIVATE, fd, 0); + if (addr_ == MAP_FAILED) { + std::cerr << "Failed to mmap file " << file_path << std::endl; + close(fd); + return; + } + + close(fd); + initialized_ = true; + offset_ = 0; + closure_ = std::make_shared(initialized_, file_size_, addr_); + }; + + ~MmapInputArchive() { + } + + bool initialized() const { + return initialized_; + } + + template + bool load(T* t, size_t align_size = 1) { + assert(offset_ + sizeof(T) <= file_size_); + char* p = (char*)addr_ + offset_; + offset_ += sizeof(T); + *t = *(reinterpret_cast(p)); + return align(align_size); + } + + char* load(size_t n, size_t align_size = 1) { + assert(offset_ + n <= file_size_); + char* p = (char*)addr_ + offset_; + offset_ += n; + align(align_size); + return p; + } + + std::shared_ptr closure() { + return closure_; + } +private: + // padding for align + bool align(size_t align) { + size_t padding_size = (offset_ + align - 1) / align * align - offset_; + if (padding_size == 0) { + return true; + } + offset_ += padding_size; + return true; + } + + bool initialized_; + size_t file_size_; + void* addr_; + size_t offset_; + std::shared_ptr closure_; +}; + +template +class MmapAllocator: public std::allocator { +public: + using value_type = T; + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using MmapClosure = typename MmapInputArchive::MmapClosure; + + MmapAllocator(): closure(nullptr) { + } + + ~MmapAllocator() { + // will call ~MmapClosure(); + } + + MmapAllocator(const MmapAllocator& m) { + this->closure = m.closure; + }; + + template + MmapAllocator(const MmapAllocator& m) { + this->closure = m.closure; + }; + + + inline pointer allocate(size_type n, const void * = 0) { + auto ret = std::allocator::allocate(n); + return ret; + } + + inline void deallocate(pointer p, size_type n) { + // mmaped memory, do not free here + if (closure + && (char*)closure->addr <= (char*)p + && (char*)p < (char*)closure->addr + closure->length) { + return; + } else { + std::allocator::deallocate(p, n); + } + } + template + struct rebind { + typedef MmapAllocator other; + }; + + void set_closure(std::shared_ptr c) { + if (closure == nullptr) { + closure = c; + } + } +public: + std::shared_ptr closure; +}; +#endif // end if __linux__ + +} // namespace phmap + +#endif // phmap_dump_h_guard_ \ No newline at end of file diff --git a/parallel_hashmap/phmap_utils.h b/parallel_hashmap/phmap_utils.h index f9c34f7..ccc078c 100644 --- a/parallel_hashmap/phmap_utils.h +++ b/parallel_hashmap/phmap_utils.h @@ -23,11 +23,8 @@ #include #include -#include -#include -#include #include "phmap_bits.h" -#include "phmap_base.h" + namespace phmap { @@ -309,107 +306,6 @@ H HashStateBase::combine(H seed, const T& v, const Ts&... vs) using HashState = HashStateBase; - -// ----------------------------------------------------------------------------- - -#define CHECK_FILE(f) { \ - if (!f.is_open()) { \ - std::cerr << "File is not open!" << std::endl; \ - return false; \ - } \ -} - -template -class ArchiveGuard { -public: - ArchiveGuard(Archive* ar): ar_(ar) { - if (ar_->guard_ == NULL) { - ar_->guard_ = this; - } - }; - ~ArchiveGuard() { - if (ar_ && ar_->guard_ == this) { - ar_->finish(); - } - } -private: - Archive* ar_; -}; - -class BinaryOutputArchive { -public: - using Guard = ArchiveGuard; - - BinaryOutputArchive(const std::string& file_path): guard_(NULL) { - ofs_.open(file_path.c_str(), std::ios_base::binary); - } - - virtual ~BinaryOutputArchive() { - finish(); - } - - bool dump(char* p, size_t sz) { - CHECK_FILE(ofs_); - ofs_.write(p, sz); - return true; - } - - template - typename std::enable_if::value, bool>::type - dump(const V& v) { - CHECK_FILE(ofs_); - ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); - return true; - } - - void finish() { - if (ofs_.is_open()) { - ofs_.close(); - } - } -private: - friend class ArchiveGuard; - std::ofstream ofs_; - Guard* guard_; -}; - - -class BinaryInputArchive { -public: - using Guard = ArchiveGuard; - - BinaryInputArchive(const std::string& file_path): guard_(NULL) { - ifs_.open(file_path.c_str(), std::ios_base::binary); - } - - virtual ~BinaryInputArchive() { - finish(); - } - - bool load(char* p, size_t sz) { - CHECK_FILE(ifs_); - ifs_.read(p, sz); - return true; - } - - template - typename std::enable_if::value, bool>::type - load(V* v) { - CHECK_FILE(ifs_); - ifs_.read(reinterpret_cast(v), sizeof(V)); - return true; - } - - void finish() { - if (ifs_.is_open()) { - ifs_.close(); - } - } -private: - friend class ArchiveGuard; - std::ifstream ifs_; - Guard* guard_; -}; } // namespace phmap diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc index e557771..da15435 100644 --- a/tests/dump_load_test.cc +++ b/tests/dump_load_test.cc @@ -1,9 +1,8 @@ #include -#include "gmock/gmock.h" #include "gtest/gtest.h" -#include "parallel_hashmap/phmap.h" +#include "parallel_hashmap/phmap_dump.h" namespace phmap { namespace container_internal { @@ -14,6 +13,9 @@ using ::phmap::flat_hash_map; using ::phmap::parallel_flat_hash_map; using ::phmap::BinaryOutputArchive; using ::phmap::BinaryInputArchive; +using ::phmap::MmapOutputArchive; +using ::phmap::MmapInputArchive; +using ::phmap::MmapAllocator; TEST(DumpLoad, FlatHashSet_uin32) { flat_hash_set st1; @@ -78,6 +80,68 @@ TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { EXPECT_EQ(1299, mp2[2130]); } +#if defined(__linux__) + +TEST(MmapDumpLoad, FlatHashMap_uint64_uint32) { + using MapType = flat_hash_map, + phmap::container_internal::hash_default_eq, + phmap::MmapAllocator< + phmap::container_internal::Pair>>; + + MapType mp1; + MmapOutputArchive ar_out("./dump.data"); + + mp1[78731] = 99; + mp1[13141] = 299; + mp1[2651] = 101; + + EXPECT_TRUE(mp1.mmap_dump(ar_out)); + MapType mp2; + MmapInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(mp2.mmap_load(ar_in)); + + EXPECT_EQ(3, mp2.size()); + EXPECT_TRUE(mp2.count(78731)); + EXPECT_TRUE(mp2.count(13141)); + EXPECT_EQ(99, mp2.at(78731)); + EXPECT_EQ(101, mp2.at(2651)); +} + +TEST(MmapDumpLoad, ParallelFlatHashMap_uint64_uint32) { + using MapType = parallel_flat_hash_map, + phmap::container_internal::hash_default_eq, + phmap::MmapAllocator< + phmap::container_internal::Pair>, + 4, + phmap::NullMutex>; + + MapType mp1; + MmapOutputArchive ar_out("./dump.data"); + + mp1[100] = 99; + mp1[300] = 299; + mp1[101] = 992; + mp1[1300] = 2991; + mp1[1130] = 299; + mp1[2130] = 1299; + + EXPECT_TRUE(mp1.mmap_dump(ar_out)); + MapType mp2; + MmapInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(mp2.mmap_load(ar_in)); + + EXPECT_EQ(6, mp2.size()); + EXPECT_EQ(99, mp2[100]); + EXPECT_EQ(299, mp2[300]); + EXPECT_EQ(299, mp2[1130]); + EXPECT_EQ(1299, mp2[2130]); +} +#endif + } } } From 3653f036838eb4eedfc5a4f1ac8ff89b56e5680f Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Sun, 25 Aug 2019 20:12:14 +0800 Subject: [PATCH 12/13] fix typo --- parallel_hashmap/phmap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 8bdb15e..404ac22 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -2329,7 +2329,7 @@ class parallel_hash_set KeyArg::value && IsTransparent::value>; static_assert(N <= 12, "N = 12 means 4096 hash tables!"); - constexpr static size_t num_tables = 1 N; + constexpr static size_t num_tables = 1 << N; constexpr static size_t mask = num_tables - 1; public: From 62a1674d9403960581804d79851d3dd45a04dfab Mon Sep 17 00:00:00 2001 From: sunkaicheng Date: Sun, 25 Aug 2019 20:44:41 +0800 Subject: [PATCH 13/13] remove mmap load and dump --- examples/dump_load.cc | 74 ------- parallel_hashmap/phmap.h | 11 -- parallel_hashmap/phmap_dump.h | 349 +--------------------------------- tests/dump_load_test.cc | 64 ------- 4 files changed, 1 insertion(+), 497 deletions(-) diff --git a/examples/dump_load.cc b/examples/dump_load.cc index efe3135..81fff62 100644 --- a/examples/dump_load.cc +++ b/examples/dump_load.cc @@ -49,84 +49,10 @@ void dump_load_parallel_flat_hash_map() { std::cout << "key: " << n.first << ", value: " << n.second << "\n"; } -#if defined(__linux__) -void mmap_load_uint64_uint32() { - using MapType = flat_hash_map, - phmap::container_internal::hash_default_eq, - phmap::MmapAllocator< - phmap::container_internal::Pair>>; - MapType mp1; - mp1.reserve(100); - phmap::MmapOutputArchive ar_out("./dump.data"); - // Add a new entry - mp1[100] = 99; - mp1[300] = 299; - - // Iterate and print keys and values - for (const auto& n : mp1) - std::cout << n.first << "'s value is: " << n.second << "\n"; - - mp1.mmap_dump(ar_out); - MapType mp2; - - phmap::MmapInputArchive ar_in("./dump.data"); - mp2.mmap_load(ar_in); - mp2[849242] = 141; - mp2[11] = 1111; - // Iterate and print keys and values g|++ - for (const auto& n : mp2) - std::cout << n.first << "'s value is: " << n.second << "\n"; -} - -void mmap_load_parallel_flat_hash_map() { - using MapType = parallel_flat_hash_map, - phmap::container_internal::hash_default_eq, - phmap::MmapAllocator< - phmap::container_internal::Pair>, - 4, - phmap::NullMutex>; - - MapType mp1; - phmap::MmapOutputArchive ar_out("./dump.data"); - - // Add a new entry - mp1[100] = 99; - mp1[300] = 299; - mp1[101] = 992; - mp1[1300] = 2991; - mp1[1130] = 299; - mp1[2130] = 1299; - // Iterate and print - for (const auto& n : mp1) - std::cout << "key: " << n.first << ", value: " << n.second << "\n"; - - mp1.mmap_dump(ar_out); - MapType mp2; - phmap::MmapInputArchive ar_in("./dump.data"); - - mp2.mmap_load(ar_in); - std::cout << "[debug] map capacity: " << mp2.capacity() << ", size: " << mp2.size() << std::endl; - for (size_t i = 0; i < 100; i ++) { - mp2[6771 + i] = i; - } - std::cout << "[debug] map capacity: " << mp2.capacity() << ", size: " << mp2.size() << std::endl; - - for (const auto& n : mp2) - std::cout << "key: " << n.first << ", value: " << n.second << "\n"; -} -#endif - int main() { dump_load_uint64_uint32(); dump_load_parallel_flat_hash_map(); - -#if defined(__linux__) - mmap_load_uint64_uint32(); - mmap_load_parallel_flat_hash_map(); -#endif return 0; } diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 404ac22..2a2bcc2 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1540,12 +1540,6 @@ public: template bool load(InputArchive&); - template - bool mmap_dump(OutputArchive&); - - template - bool mmap_load(MmapInputArchive&); - void rehash(size_t n) { if (n == 0 && capacity_ == 0) return; if (n == 0 && size_ == 0) { @@ -3157,11 +3151,6 @@ public: template bool load(InputArchive& ar); - template - bool mmap_dump(OutputArchive& ar); - - template - bool mmap_load(InputArchive& ar); private: template friend struct phmap::container_internal::hashtable_debug_internal::HashtableDebugAccess; diff --git a/parallel_hashmap/phmap_dump.h b/parallel_hashmap/phmap_dump.h index b75262a..bd36ac9 100644 --- a/parallel_hashmap/phmap_dump.h +++ b/parallel_hashmap/phmap_dump.h @@ -111,79 +111,6 @@ bool raw_hash_set::load(InputArchive& ar) { return true; } - -template -template -bool raw_hash_set::mmap_dump(OutputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable::value, - "value_type should be dumpable"); - - typename OutputArchive::Guard guard(&ar); - size_t align_size = Layout::Alignment(); - if (!ar.dump(size_, align_size)) { - std::cerr << "Failed to dump size_" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.dump(capacity_, align_size)) { - std::cerr << "Failed to dump capacity_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast(ctrl_), - sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1), align_size)) { - - std::cerr << "Failed to dump ctrl_" << std::endl; - return false; - } - if (!ar.dump(reinterpret_cast(slots_), - sizeof(slot_type) * capacity_, align_size)) { - std::cerr << "Failed to dump slot_" << std::endl; - return false; - } - return true; -} - -template -template -bool raw_hash_set::mmap_load(MmapInputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable::value, - "value_type should be dumpable"); - - assert(ar.initialized()); - auto closure = ar.closure(); - this->alloc_ref().set_closure(closure); - size_t align_size = Layout::Alignment(); - if (!ar.load(&size_, align_size)) { - std::cerr << "Failed to load size!" << std::endl; - return false; - } - if (size_ == 0) { - return true; - } - if (!ar.load(&capacity_, align_size)) { - std::cerr << "Failed to load capacity!" << std::endl; - return false; - } - - - if (std::is_same>::value) { - infoz_ = Sample(); - } - reset_growth_left(); - infoz_.RecordStorageChanged(size_, capacity_); - - char* p_ctrl = ar.load( - sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1), align_size); - ctrl_ = reinterpret_cast(p_ctrl); - - char* p_slots = ar.load(sizeof(slot_type) * capacity_, align_size); - slots_ = reinterpret_cast(p_slots); - return true; -} - - ////// parallel_hash_set template class RefSet, @@ -241,67 +168,6 @@ bool parallel_hash_set::load(InputArch } return true; } - -template class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -template -bool parallel_hash_set::mmap_dump(OutputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable::value, - "value_type should be dumpable"); - - typename OutputArchive::Guard guard(&ar); - size_t align_size = EmbeddedSet::Layout::Alignment(); - if (! ar.dump(subcnt(), align_size)) { - std::cerr << "Failed to dump meta!" << std::endl; - return false; - } - for (size_t i = 0; i < sets_.size(); ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.mmap_dump(ar)) { - std::cerr << "Failed to dump submap " << i << std::endl; - return false; - } - } - return true; -} - -template class RefSet, - class Mtx_, - class Policy, class Hash, class Eq, class Alloc> -template -bool parallel_hash_set::mmap_load(InputArchive& ar) { - static_assert(type_traits_internal::IsTriviallyCopyable::value, - "value_type should be dumpable"); - - assert(ar.initialized()); - auto closure = ar.closure(); - this->alloc_ref().set_closure(closure); - size_t submap_count = 0; - size_t align_size = EmbeddedSet::Layout::Alignment(); - if (!ar.load(&submap_count, align_size)) { - std::cerr << "Failed to load submap count!" << std::endl; - return false; - } - - if (submap_count != subcnt()) { - std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; - return false; - } - - for (size_t i = 0; i < submap_count; ++i) { - auto& inner = sets_[i]; - typename Lockable::UniqueLock m(const_cast(inner)); - if (!inner.set_.mmap_load(ar)) { - std::cerr << "Failed to load submap " << i << std::endl; - return false; - } - } - return true; -} } // namesapce container_internal @@ -417,219 +283,6 @@ private: Guard* guard_; }; -#if defined(__linux__) // only support linux's mmap now -// mmap dump && mmap load -#include -#include -#include -#include - -class MmapOutputArchive { -public: - using Guard = ArchiveGuard; - - MmapOutputArchive(const std::string& file_path): offset_(0), guard_(NULL) { - ofs_.open(file_path.c_str(), std::ios_base::binary); - } - - virtual ~MmapOutputArchive() { - finish(); - } - - bool dump(char* p, size_t sz, size_t align_size) { - CHECK_FILE(ofs_); - ofs_.write(p, sz); - offset_ += sz; - return align(align_size); - } - - template - typename std::enable_if::value, bool>::type - dump(const V& v, size_t align_size) { - CHECK_FILE(ofs_); - ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); - offset_ += sizeof(V); - return align(align_size); - } - - void finish() { - if (ofs_.is_open()) { - ofs_.close(); - offset_ = 0; - } - } -private: - // padding for align - bool align(size_t align) { - size_t padding_size = (offset_ + align - 1) / align * align - offset_; - if (padding_size == 0) { - return true; - } - std::string padding(padding_size, '\0'); - ofs_.write(padding.c_str(), padding_size); - offset_ += padding_size; - return true; - } - - friend class ArchiveGuard; - std::ofstream ofs_; - size_t offset_; - Guard* guard_; -}; - - -class MmapInputArchive { -public: - class MmapClosure { - public: - MmapClosure(bool init = false, size_t len = 0, void* a = NULL): - initialized(init), length(len), addr(a) {}; - - ~MmapClosure() { - if (initialized && addr != MAP_FAILED) { - munmap(addr, length); - } - } - bool initialized; - size_t length; - void* addr; - }; - - MmapInputArchive(const std::string& file_path): - initialized_(false), closure_(nullptr) { - int fd = open(file_path.c_str(), O_RDONLY); - if (fd == -1) { - std::cerr << "Failed to open file " << file_path << std::endl; - return; - } - - struct stat st; - if (fstat(fd, &st) == -1) { - std::cerr << "Failed to stat file " << file_path << std::endl; - close(fd); - return; - } - - file_size_ = st.st_size; - - addr_ = mmap(NULL, file_size_, PROT_READ|PROT_WRITE, - MAP_PRIVATE, fd, 0); - if (addr_ == MAP_FAILED) { - std::cerr << "Failed to mmap file " << file_path << std::endl; - close(fd); - return; - } - - close(fd); - initialized_ = true; - offset_ = 0; - closure_ = std::make_shared(initialized_, file_size_, addr_); - }; - - ~MmapInputArchive() { - } - - bool initialized() const { - return initialized_; - } - - template - bool load(T* t, size_t align_size = 1) { - assert(offset_ + sizeof(T) <= file_size_); - char* p = (char*)addr_ + offset_; - offset_ += sizeof(T); - *t = *(reinterpret_cast(p)); - return align(align_size); - } - - char* load(size_t n, size_t align_size = 1) { - assert(offset_ + n <= file_size_); - char* p = (char*)addr_ + offset_; - offset_ += n; - align(align_size); - return p; - } - - std::shared_ptr closure() { - return closure_; - } -private: - // padding for align - bool align(size_t align) { - size_t padding_size = (offset_ + align - 1) / align * align - offset_; - if (padding_size == 0) { - return true; - } - offset_ += padding_size; - return true; - } - - bool initialized_; - size_t file_size_; - void* addr_; - size_t offset_; - std::shared_ptr closure_; -}; - -template -class MmapAllocator: public std::allocator { -public: - using value_type = T; - using pointer = T*; - using const_pointer = const T*; - using reference = T&; - using const_reference = const T&; - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - using MmapClosure = typename MmapInputArchive::MmapClosure; - - MmapAllocator(): closure(nullptr) { - } - - ~MmapAllocator() { - // will call ~MmapClosure(); - } - - MmapAllocator(const MmapAllocator& m) { - this->closure = m.closure; - }; - - template - MmapAllocator(const MmapAllocator& m) { - this->closure = m.closure; - }; - - - inline pointer allocate(size_type n, const void * = 0) { - auto ret = std::allocator::allocate(n); - return ret; - } - - inline void deallocate(pointer p, size_type n) { - // mmaped memory, do not free here - if (closure - && (char*)closure->addr <= (char*)p - && (char*)p < (char*)closure->addr + closure->length) { - return; - } else { - std::allocator::deallocate(p, n); - } - } - template - struct rebind { - typedef MmapAllocator other; - }; - - void set_closure(std::shared_ptr c) { - if (closure == nullptr) { - closure = c; - } - } -public: - std::shared_ptr closure; -}; -#endif // end if __linux__ - } // namespace phmap -#endif // phmap_dump_h_guard_ \ No newline at end of file +#endif // phmap_dump_h_guard_ diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc index d3f9b5f..4933406 100644 --- a/tests/dump_load_test.cc +++ b/tests/dump_load_test.cc @@ -13,9 +13,6 @@ using ::phmap::flat_hash_map; using ::phmap::parallel_flat_hash_map; using ::phmap::BinaryOutputArchive; using ::phmap::BinaryInputArchive; -using ::phmap::MmapOutputArchive; -using ::phmap::MmapInputArchive; -using ::phmap::MmapAllocator; TEST(DumpLoad, FlatHashSet_uin32) { flat_hash_set st1; @@ -79,67 +76,6 @@ TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { EXPECT_EQ(1299, mp2[2130]); } -#if defined(__linux__) - -TEST(MmapDumpLoad, FlatHashMap_uint64_uint32) { - using MapType = flat_hash_map, - phmap::container_internal::hash_default_eq, - phmap::MmapAllocator< - phmap::container_internal::Pair>>; - - MapType mp1; - MmapOutputArchive ar_out("./dump.data"); - - mp1[78731] = 99; - mp1[13141] = 299; - mp1[2651] = 101; - - EXPECT_TRUE(mp1.mmap_dump(ar_out)); - MapType mp2; - MmapInputArchive ar_in("./dump.data"); - - EXPECT_TRUE(mp2.mmap_load(ar_in)); - - EXPECT_EQ(3, mp2.size()); - EXPECT_TRUE(mp2.count(78731)); - EXPECT_TRUE(mp2.count(13141)); - EXPECT_EQ(99, mp2.at(78731)); - EXPECT_EQ(101, mp2.at(2651)); -} - -TEST(MmapDumpLoad, ParallelFlatHashMap_uint64_uint32) { - using MapType = parallel_flat_hash_map, - phmap::container_internal::hash_default_eq, - phmap::MmapAllocator< - phmap::container_internal::Pair>, - 4, - phmap::NullMutex>; - - MapType mp1; - MmapOutputArchive ar_out("./dump.data"); - - mp1[100] = 99; - mp1[300] = 299; - mp1[101] = 992; - mp1[1300] = 2991; - mp1[1130] = 299; - mp1[2130] = 1299; - - EXPECT_TRUE(mp1.mmap_dump(ar_out)); - MapType mp2; - MmapInputArchive ar_in("./dump.data"); - - EXPECT_TRUE(mp2.mmap_load(ar_in)); - EXPECT_EQ(6, mp2.size()); - EXPECT_EQ(99, mp2[100]); - EXPECT_EQ(299, mp2[300]); - EXPECT_EQ(299, mp2[1130]); - EXPECT_EQ(1299, mp2[2130]); -} -#endif - } } }