diff --git a/.gitignore b/.gitignore index 318545c..b208e24 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ benchmark/output benchmark/charts.html build* .vagrant +**/.vscode TAGS diff --git a/CMakeLists.txt b/CMakeLists.txt index 072802e..15017bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ set(CMAKE_SUPPRESS_REGENERATION true) ## suppress ZERO_CHECK project include(GNUInstallDirs) include(CMakePackageConfigHelpers) include(helpers) +include_directories("${CMAKE_CURRENT_SOURCE_DIR}") add_library(${PROJECT_NAME} INTERFACE) @@ -110,6 +111,10 @@ if (PHMAP_BUILD_TESTS) phmap_cc_test(NAME parallel_flat_hash_map_mutex SRCS "tests/parallel_flat_hash_map_mutex_test.cc" COPTS "-DUNORDERED_MAP_CXX17" DEPS gmock_main) + phmap_cc_test(NAME dump_load SRCS "tests/dump_load_test.cc" + COPTS "-DUNORDERED_MAP_CXX17" DEPS gmock_main) + + endif() if (PHMAP_BUILD_EXAMPLES) @@ -130,6 +135,8 @@ if (PHMAP_BUILD_EXAMPLES) add_executable(ex_two_files examples/f1.cc examples/f2.cc phmap.natvis) add_executable(ex_insert_bench examples/insert_bench.cc phmap.natvis) add_executable(ex_knucleotide examples/knucleotide.cc phmap.natvis) + add_executable(ex_dump_load examples/dump_load.cc phmap.natvis) + target_link_libraries(ex_knucleotide Threads::Threads) target_link_libraries(ex_bench Threads::Threads) endif() diff --git a/examples/dump_load.cc b/examples/dump_load.cc new file mode 100644 index 0000000..81fff62 --- /dev/null +++ b/examples/dump_load.cc @@ -0,0 +1,58 @@ +#include +#include +#include + +using phmap::flat_hash_map; +using phmap::parallel_flat_hash_map; + +void dump_load_uint64_uint32() { + flat_hash_map mp1; + phmap::BinaryOutputArchive ar_out("./dump.data"); + // Add a new entry + mp1[100] = 99; + mp1[300] = 299; + + // Iterate and print keys and values + for (const auto& n : mp1) + std::cout << n.first << "'s value is: " << n.second << "\n"; + + mp1.dump(ar_out); + flat_hash_map mp2; + phmap::BinaryInputArchive ar_in("./dump.data"); + mp2.load(ar_in); + // Iterate and print keys and values g|++ + for (const auto& n : mp2) + std::cout << n.first << "'s value is: " << n.second << "\n"; +} + +void dump_load_parallel_flat_hash_map() { + parallel_flat_hash_map mp1; + phmap::BinaryOutputArchive ar_out("./dump.data"); + + // Add a new entry + mp1[100] = 99; + mp1[300] = 299; + mp1[101] = 992; + mp1[1300] = 2991; + mp1[1130] = 299; + mp1[2130] = 1299; + // Iterate and print + for (const auto& n : mp1) + std::cout << "key: " << n.first << ", value: " << n.second << "\n"; + + mp1.dump(ar_out); + parallel_flat_hash_map mp2; + phmap::BinaryInputArchive ar_in("./dump.data"); + + mp2.load(ar_in); + for (const auto& n : mp2) + std::cout << "key: " << n.first << ", value: " << n.second << "\n"; +} + +int main() +{ + dump_load_uint64_uint32(); + dump_load_parallel_flat_hash_map(); + return 0; +} + diff --git a/examples/serialize.cc b/examples/serialize.cc index db4adc7..3c2d51b 100644 --- a/examples/serialize.cc +++ b/examples/serialize.cc @@ -22,9 +22,9 @@ void showtime(const char *name, std::function doit) doit(); auto t2 = std::chrono::high_resolution_clock::now(); auto elapsed = milliseconds(t2 - t1).count(); - printf("%s: %.3fms\n", name, (int)elapsed / 1000.0f); + printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f); } - + int main() { using MapType = phmap::flat_hash_map, int>; diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 3957568..d51ec01 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -1534,6 +1534,12 @@ public: } } + template + bool dump(OutputArchive&); + + template + bool load(InputArchive&); + void rehash(size_t n) { if (n == 0 && capacity_ == 0) return; if (n == 0 && size_ == 0) { @@ -3141,6 +3147,12 @@ public: a.swap(b); } + template + bool dump(OutputArchive& ar); + + template + bool load(InputArchive& ar); + private: template friend struct phmap::container_internal::hashtable_debug_internal::HashtableDebugAccess; diff --git a/parallel_hashmap/phmap_dump.h b/parallel_hashmap/phmap_dump.h new file mode 100644 index 0000000..bd36ac9 --- /dev/null +++ b/parallel_hashmap/phmap_dump.h @@ -0,0 +1,288 @@ +#if !defined(phmap_dump_h_guard_) +#define phmap_dump_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com +// +// providing dump/load/mmap_load +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +#include +#include +#include +#include +#include +#include "phmap.h" +namespace phmap +{ + +namespace type_traits_internal { + +#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801 + template struct IsTriviallyCopyable : public std::integral_constant {}; +#else + template struct IsTriviallyCopyable : public std::is_trivially_copyable {}; +#endif + +template +struct IsTriviallyCopyable> { + static constexpr bool value = IsTriviallyCopyable::value && IsTriviallyCopyable::value; +}; +} + +namespace container_internal { + +//// raw_hash_set +template +template +bool raw_hash_set::dump(OutputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename OutputArchive::Guard guard(&ar); + if (!ar.dump(size_)) { + std::cerr << "Failed to dump size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.dump(capacity_)) { + std::cerr << "Failed to dump capacity_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { + + std::cerr << "Failed to dump ctrl_" << std::endl; + return false; + } + if (!ar.dump(reinterpret_cast(slots_), + sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to dump slot_" << std::endl; + return false; + } + return true; +} + +template +template +bool raw_hash_set::load(InputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename InputArchive::Guard guard(&ar); + if (!ar.load(&size_)) { + std::cerr << "Failed to load size_" << std::endl; + return false; + } + if (size_ == 0) { + return true; + } + if (!ar.load(&capacity_)) { + std::cerr << "Failed to load capacity_" << std::endl; + return false; + } + + // allocate memory for ctrl_ and slots_ + initialize_slots(); + if (!ar.load(reinterpret_cast(ctrl_), + sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) { + std::cerr << "Failed to load ctrl" << std::endl; + return false; + } + if (!ar.load(reinterpret_cast(slots_), + sizeof(slot_type) * capacity_)) { + std::cerr << "Failed to load slot" << std::endl; + return false; + } + return true; +} + +////// parallel_hash_set +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template +bool parallel_hash_set::dump(OutputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename OutputArchive::Guard guard(&ar); + if (! ar.dump(subcnt())) { + std::cerr << "Failed to dump meta!" << std::endl; + return false; + } + for (size_t i = 0; i < sets_.size(); ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.dump(ar)) { + std::cerr << "Failed to dump submap " << i << std::endl; + return false; + } + } + return true; +} + +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +template +bool parallel_hash_set::load(InputArchive& ar) { + static_assert(type_traits_internal::IsTriviallyCopyable::value, + "value_type should be dumpable"); + + typename InputArchive::Guard guard(&ar); + size_t submap_count = 0; + if (!ar.load(&submap_count)) { + std::cerr << "Failed to load submap count!" << std::endl; + return false; + } + + if (submap_count != subcnt()) { + std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl; + return false; + } + + for (size_t i = 0; i < submap_count; ++i) { + auto& inner = sets_[i]; + typename Lockable::UniqueLock m(const_cast(inner)); + if (!inner.set_.load(ar)) { + std::cerr << "Failed to load submap " << i << std::endl; + return false; + } + } + return true; +} +} // namesapce container_internal + + + +// ArchiveOutput & ArchiveInput + +#define CHECK_FILE(f) { \ + if (!f.is_open()) { \ + std::cerr << "File is not open!" << std::endl; \ + return false; \ + } \ +} + +template +class ArchiveGuard { +public: + ArchiveGuard(Archive* ar): ar_(ar) { + if (ar_->guard_ == NULL) { + ar_->guard_ = this; + } + }; + ~ArchiveGuard() { + if (ar_ && ar_->guard_ == this) { + ar_->finish(); + } + } +private: + Archive* ar_; +}; + +class BinaryOutputArchive { +public: + using Guard = ArchiveGuard; + + BinaryOutputArchive(const std::string& file_path): offset_(0), guard_(NULL) { + ofs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~BinaryOutputArchive() { + finish(); + } + + bool dump(char* p, size_t sz) { + CHECK_FILE(ofs_); + ofs_.write(p, sz); + offset_ += sz; + return true; + } + + template + typename std::enable_if::value, bool>::type + dump(const V& v) { + CHECK_FILE(ofs_); + ofs_.write(reinterpret_cast(const_cast(&v)), sizeof(V)); + offset_ += sizeof(V); + return true; + } + + void finish() { + if (ofs_.is_open()) { + ofs_.close(); + offset_ = 0; + } + } + +private: + friend class ArchiveGuard; + std::ofstream ofs_; + size_t offset_; + Guard* guard_; +}; + + +class BinaryInputArchive { +public: + using Guard = ArchiveGuard; + + BinaryInputArchive(const std::string& file_path): offset_(0), guard_(NULL) { + ifs_.open(file_path.c_str(), std::ios_base::binary); + } + + virtual ~BinaryInputArchive() { + finish(); + } + + bool load(char* p, size_t sz) { + CHECK_FILE(ifs_); + ifs_.read(p, sz); + offset_ += sz; + return true; + } + + template + typename std::enable_if::value, bool>::type + load(V* v) { + CHECK_FILE(ifs_); + ifs_.read(reinterpret_cast(v), sizeof(V)); + offset_ += sizeof(V); + return true; + } + + void finish() { + if (ifs_.is_open()) { + ifs_.close(); + offset_ = 0; + } + } + +private: + friend class ArchiveGuard; + std::ifstream ifs_; + size_t offset_; + Guard* guard_; +}; + +} // namespace phmap + +#endif // phmap_dump_h_guard_ diff --git a/tests/dump_load_test.cc b/tests/dump_load_test.cc new file mode 100644 index 0000000..4933406 --- /dev/null +++ b/tests/dump_load_test.cc @@ -0,0 +1,82 @@ +#include + +#include "gtest/gtest.h" + +#include "parallel_hashmap/phmap_dump.h" + +namespace phmap { +namespace container_internal { +namespace { + +using ::phmap::flat_hash_set; +using ::phmap::flat_hash_map; +using ::phmap::parallel_flat_hash_map; +using ::phmap::BinaryOutputArchive; +using ::phmap::BinaryInputArchive; + +TEST(DumpLoad, FlatHashSet_uin32) { + flat_hash_set st1; + BinaryOutputArchive ar_out("./dump.data"); + + st1.insert(1991); + st1.insert(1202); + + EXPECT_TRUE(st1.dump(ar_out)); + flat_hash_set st2; + BinaryInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(st2.load(ar_in)); + + EXPECT_EQ(2, st2.size()); + EXPECT_TRUE(st2.count(1991)); + EXPECT_TRUE(st2.count(1202)); +} + +TEST(DumpLoad, FlatHashMap_uint64_uint32) { + flat_hash_map mp1; + BinaryOutputArchive ar_out("./dump.data"); + + mp1[78731] = 99; + mp1[13141] = 299; + mp1[2651] = 101; + + EXPECT_TRUE(mp1.dump(ar_out)); + flat_hash_map mp2; + BinaryInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(mp2.load(ar_in)); + + EXPECT_EQ(3, mp2.size()); + EXPECT_TRUE(mp2.count(78731)); + EXPECT_TRUE(mp2.count(13141)); + EXPECT_EQ(99, mp2.at(78731)); + EXPECT_EQ(101, mp2.at(2651)); +} + +TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) { + parallel_flat_hash_map mp1; + BinaryOutputArchive ar_out("./dump.data"); + + mp1[100] = 99; + mp1[300] = 299; + mp1[101] = 992; + mp1[1300] = 2991; + mp1[1130] = 299; + mp1[2130] = 1299; + + EXPECT_TRUE(mp1.dump(ar_out)); + parallel_flat_hash_map mp2; + BinaryInputArchive ar_in("./dump.data"); + + EXPECT_TRUE(mp2.load(ar_in)); + EXPECT_EQ(6, mp2.size()); + EXPECT_EQ(99, mp2[100]); + EXPECT_EQ(299, mp2[300]); + EXPECT_EQ(299, mp2[1130]); + EXPECT_EQ(1299, mp2[2130]); +} + +} +} +} +