diff --git a/.gitignore b/.gitignore index c09a4b2..a6d3cf8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ benchmark/charts.html build build_linux .vagrant +**/.vscode TAGS diff --git a/CMakeLists.txt b/CMakeLists.txt index 072802e..12e46c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ set(CMAKE_SUPPRESS_REGENERATION true) ## suppress ZERO_CHECK project include(GNUInstallDirs) include(CMakePackageConfigHelpers) include(helpers) +include_directories("${CMAKE_CURRENT_SOURCE_DIR}") add_library(${PROJECT_NAME} INTERFACE) @@ -130,6 +131,8 @@ if (PHMAP_BUILD_EXAMPLES) add_executable(ex_two_files examples/f1.cc examples/f2.cc phmap.natvis) add_executable(ex_insert_bench examples/insert_bench.cc phmap.natvis) add_executable(ex_knucleotide examples/knucleotide.cc phmap.natvis) + add_executable(ex_dump_load examples/dump_load.cc phmap.natvis) + target_link_libraries(ex_knucleotide Threads::Threads) target_link_libraries(ex_bench Threads::Threads) endif() diff --git a/examples/dump_load.cc b/examples/dump_load.cc new file mode 100644 index 0000000..f4ae0bd --- /dev/null +++ b/examples/dump_load.cc @@ -0,0 +1,138 @@ +#include +#include +#include + +using phmap::flat_hash_map; +using phmap::flat_hash_set; + +void load_dump_string_string() { + flat_hash_map mp1; + + // Add a new entry + mp1["key-1"] = "value-1"; + mp1["key-2"] = "value-2"; + + // Iterate and print keys and values + for (const auto& n : mp1) + std::cout << n.first << "'s value is: " << n.second << "\n"; + + mp1.dump("./dump.data"); + flat_hash_map mp2; + + mp2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : mp2) + std::cout << n.first << "'s value is: " << n.second << "\n"; +} + +void load_dump_uint64_uint32() { + flat_hash_map mp1; + + // Add a new entry + mp1[100] = 99; + mp1[300] = 299; + + // Iterate and print keys and values + for (const auto& n : mp1) + std::cout << n.first << "'s value is: " << n.second << "\n"; + + mp1.dump("./dump.data"); + flat_hash_map mp2; + + mp2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : mp2) + std::cout << n.first << "'s value is: " << n.second << "\n"; +} + +void load_dump_string_uint32() { + flat_hash_map mp1; + + // Add a new entry + mp1["key-1"] = 99; + mp1["key-2"] = 299; + + // Iterate and print keys and values + for (const auto& n : mp1) + std::cout << n.first << "'s value is: " << n.second << "\n"; + + mp1.dump("./dump.data"); + flat_hash_map mp2; + + mp2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : mp2) + std::cout << n.first << "'s value is: " << n.second << "\n"; +} + +void load_dump_uint32_string() { + flat_hash_map mp1; + + // Add a new entry + mp1[100] = "hello"; + mp1[299] = "world"; + + // Iterate and print keys and values + for (const auto& n : mp1) + std::cout << n.first << "'s value is: " << n.second << "\n"; + + mp1.dump("./dump.data"); + flat_hash_map mp2; + + mp2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : mp2) + std::cout << n.first << "'s value is: " << n.second << "\n"; +} + +void load_dump_string() { + flat_hash_set st1; + + // Add a new entry + st1.insert("hello"); + st1.insert("world"); + + // Iterate and print + for (const auto& n : st1) + std::cout << "value: " << n << "\n"; + + st1.dump("./dump.data"); + flat_hash_set st2; + + st2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : st2) + std::cout << "value: " << n << "\n"; +} + +void load_dump_uint64() { + flat_hash_set st1; + + // Add a new entry + st1.insert(878); + st1.insert(1424); + + // Iterate and print + for (const auto& n : st1) + std::cout << "value: " << n << "\n"; + + st1.dump("./dump.data"); + flat_hash_set st2; + + st2.load("./dump.data"); + // Iterate and print keys and values g|++ + for (const auto& n : st2) + std::cout << "value: " << n << "\n"; +} + +int main() +{ + load_dump_string_string(); + load_dump_uint64_uint32(); + load_dump_string_uint32(); + load_dump_uint32_string(); + load_dump_string(); + load_dump_uint64(); + std::remove("./dump.data"); + return 0; +} \ No newline at end of file diff --git a/parallel_hashmap/phmap.h b/parallel_hashmap/phmap.h index 94ed883..e715734 100644 --- a/parallel_hashmap/phmap.h +++ b/parallel_hashmap/phmap.h @@ -45,6 +45,9 @@ #include #include #include +#include +#include +#include #include "phmap_utils.h" #include "phmap_base.h" @@ -1534,6 +1537,151 @@ public: } } + template + typename std::enable_if::value, bool>::type + dump(const std::string& dump_file) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + if (size_ == 0) { + std::cout << "Empty set, nothing to dump" << std::endl; + return true; + } + assert(slots_ != nullptr); + std::ofstream ofs(dump_file); + if (!ofs.is_open()) { + std::cout << "Failed to open dump file " << dump_file << std::endl; + return false; + } + ofs.write(reinterpret_cast(&size_), sizeof(size_)); + ofs.write(reinterpret_cast(&capacity_), sizeof(capacity_)); + ofs.write(reinterpret_cast(ctrl_), capacity_ * sizeof(ctrl_t)); + ofs.write(reinterpret_cast(slots_), capacity_ * sizeof(slot_type)); + ofs.close(); + return true; + } + + template + typename std::enable_if::value, bool>::type + load(const std::string& load_file) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + std::ifstream ifs(load_file); + if (!ifs.is_open()) { + std::cerr << "Failed to open load file " << load_file << std::endl; + return false; + } + // get file size + ifs.seekg(0, std::ios::end); + size_t file_size = ifs.tellg(); + ifs.seekg(0, std::ios::beg); + if (file_size <= sizeof(size_) + sizeof(capacity_)) { + std::cerr << "Invalid file format. file size: " << file_size << ", size_: " + << size_ << ", capacity_: " << capacity_ << ", slot type size: " + << sizeof(slot_type); + return false; + } + + ifs.read(reinterpret_cast(&size_), sizeof(size_)); + ifs.read(reinterpret_cast(&capacity_), sizeof(capacity_)); + if (file_size != sizeof(size_) + sizeof(capacity_) + capacity_ * sizeof(ctrl_t) + + capacity_ * sizeof(slot_type)) { + std::cerr << "Invalid file format. file size: " << file_size << ", size_: " + << size_ << ", capacity_: " << capacity_ << ", slot type size: " + << sizeof(slot_type); + return false; + } + // allocate memory for ctrl_ and slots_ + initialize_slots(); + + ifs.read(reinterpret_cast(ctrl_), capacity_ * sizeof(ctrl_t)); + ifs.read(reinterpret_cast(slots_), capacity_ * sizeof(slot_type)); + ifs.close(); + return true; + } + + // V will be V for hash_set and std::pair for hash_map + template + typename std::enable_if::value + && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + dump(const std::string& dump_file) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + if (size_ == 0) { + std::cout << "Empty set, nothing to dump" << std::endl; + return true; + } + assert(slots_ != nullptr); + std::ofstream ofs(dump_file); + if (!ofs.is_open()) { + std::cout << "Failed to open dump file " << dump_file << std::endl; + return false; + } + + ofs.write(reinterpret_cast(&size_), sizeof(size_)); + + for (auto it = this->begin(); it != this->end(); ++it) { + type_traits_internal::Archive::dump(*it, &ofs); + } + ofs.close(); + return true; + } + + template + typename std::enable_if::value + && type_traits_internal::IsStringOrArithmeticType::value, bool>::type + load(const std::string& load_file = "") noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + + std::ifstream ifs(load_file); + if (!ifs.is_open()) { + std::cerr << "Failed to open load file " << load_file << std::endl; + return false; + } + + size_t total_count = 0; + ifs.read((char*)&total_count, sizeof(total_count)); + + for (size_t i = 0; i < total_count; i ++) { + if (ifs.eof()) { + std::cerr << "Data is not enough, total_count: " << total_count + << ", meet eof at index: " << i << std::endl; + return false; + } + V v; + type_traits_internal::Archive::load(ifs, &v); + this->insert(v); + } + ifs.close(); + return true; + } + + template + typename std::enable_if::value, bool>::type + dump(const std::string&) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + std::cerr << "Does not support this type now!" << std::endl; + std::abort(); + return false; + } + + template + typename std::enable_if::value, bool>::type + load(const std::string&) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + std::cerr << "Does not support this type now!" << std::endl; + std::abort(); + return false; + } + void rehash(size_t n) { if (n == 0 && capacity_ == 0) return; if (n == 0 && size_ == 0) { diff --git a/parallel_hashmap/phmap_base.h b/parallel_hashmap/phmap_base.h index bbc6712..2045d2a 100644 --- a/parallel_hashmap/phmap_base.h +++ b/parallel_hashmap/phmap_base.h @@ -33,7 +33,6 @@ // See the License for the specific language governing permissions and // limitations under the License. // --------------------------------------------------------------------------- - #include #include #include @@ -71,6 +70,87 @@ struct EqualTo namespace type_traits_internal { +template +struct PairTrait : public std::false_type { + using first_type = typename std::remove_cv::type; + using second_type = typename std::remove_cv::type; +}; + +template +struct PairTrait>: public std::true_type { + using first_type = typename std::remove_cv::type; + using second_type = typename std::remove_cv::type; +}; + +template +struct IsArithmeticType { + static constexpr bool value = std::is_arithmetic::value + || (PairTrait::value && + std::is_arithmetic::first_type>::value + && std::is_arithmetic::second_type>::value); +}; + +template +struct IsStringOrArithmeticType { + static constexpr bool value = IsArithmeticType::value + || std::is_same::value + || (PairTrait::value + && (std::is_arithmetic::first_type>::value + || std::is_same::first_type, std::string>::value) + && (std::is_arithmetic::second_type>::value + || std::is_same::second_type, std::string>::value)); +}; + +// only support std::is_arithmetic or std::string types +template +struct Archive { + template + static typename std::enable_if::value, void>::type + dump(const V& v, std::ofstream* ofs) { + ofs->write(reinterpret_cast(const_cast(&v)), sizeof(V)); + } + + template + static typename std::enable_if::value, void>::type + load(std::ifstream& ifs, V* v) { + ifs.read(reinterpret_cast(v), sizeof(V)); + } + + template + static typename std::enable_if::type>::value, void>::type + dump(const V& v, std::ofstream* ofs) { + uint32_t sz = v.length(); + ofs->write(reinterpret_cast(&sz), sizeof(sz)); + ofs->write(const_cast(v.data()), sz); + } + + template + static typename std::enable_if::type>::value, void>::type + load(std::ifstream& ifs, V* v) { + uint32_t sz = 0; + ifs.read(reinterpret_cast(&sz), sizeof(sz)); + const_cast(v)->resize(sz); + ifs.read(const_cast(v->data()), sz); + } + + template + static typename std::enable_if::value && IsStringOrArithmeticType::value, void>::type + dump(const V& v, std::ofstream* ofs) { + dump::first_type>(v.first, ofs); + dump::second_type>(v.second, ofs); + } + + template + static typename std::enable_if::value && IsStringOrArithmeticType::value, void>::type + load(std::ifstream& ifs, V* v) { + using first_type = typename PairTrait::first_type; + using second_type = typename PairTrait::second_type; + load(ifs, const_cast(&v->first)); + load(ifs, const_cast(&v->second)); + } +}; + + template struct VoidTImpl { using type = void;