Merge pull request #23 from gtarcoder/master

Added dump&&load interface for trivially_copyable types

.gitignore

@@ -4,4 +4,5 @@ benchmark/output
 benchmark/charts.html
 build*
 .vagrant
+**/.vscode
 TAGS

CMakeLists.txt

@@ -17,6 +17,7 @@ set(CMAKE_SUPPRESS_REGENERATION true)            ## suppress ZERO_CHECK project
 include(GNUInstallDirs)
 include(CMakePackageConfigHelpers)
 include(helpers)
+include_directories("${CMAKE_CURRENT_SOURCE_DIR}")
 
 add_library(${PROJECT_NAME} INTERFACE)
 
@@ -110,6 +111,10 @@ if (PHMAP_BUILD_TESTS)
     phmap_cc_test(NAME parallel_flat_hash_map_mutex SRCS "tests/parallel_flat_hash_map_mutex_test.cc"
                   COPTS "-DUNORDERED_MAP_CXX17" DEPS gmock_main)
 
+    phmap_cc_test(NAME dump_load SRCS "tests/dump_load_test.cc"
+                   COPTS "-DUNORDERED_MAP_CXX17" DEPS gmock_main)
+
+
 endif()
 
 if (PHMAP_BUILD_EXAMPLES)
@@ -130,6 +135,8 @@ if (PHMAP_BUILD_EXAMPLES)
     add_executable(ex_two_files examples/f1.cc examples/f2.cc phmap.natvis)
     add_executable(ex_insert_bench examples/insert_bench.cc phmap.natvis)
     add_executable(ex_knucleotide examples/knucleotide.cc phmap.natvis)
+    add_executable(ex_dump_load examples/dump_load.cc phmap.natvis)
+
     target_link_libraries(ex_knucleotide Threads::Threads)
     target_link_libraries(ex_bench Threads::Threads)
 endif()

examples/dump_load.cc

@@ -0,0 +1,58 @@
+#include <iostream>
+#include <string>
+#include <parallel_hashmap/phmap_dump.h>
+
+using phmap::flat_hash_map;
+using phmap::parallel_flat_hash_map;
+
+void dump_load_uint64_uint32() {
+    flat_hash_map<uint64_t, uint32_t> mp1;
+    phmap::BinaryOutputArchive ar_out("./dump.data");
+    // Add a new entry
+    mp1[100] = 99;
+    mp1[300] = 299;
+
+    // Iterate and print keys and values 
+    for (const auto& n : mp1)
+        std::cout << n.first << "'s value is: " << n.second << "\n";
+ 
+    mp1.dump(ar_out);
+    flat_hash_map<uint64_t, uint32_t> mp2;
+    phmap::BinaryInputArchive ar_in("./dump.data");
+    mp2.load(ar_in);
+    // Iterate and print keys and values g|++
+    for (const auto& n : mp2)
+        std::cout << n.first << "'s value is: " << n.second << "\n";
+}
+
+void dump_load_parallel_flat_hash_map() {
+    parallel_flat_hash_map<uint64_t, uint32_t> mp1;
+    phmap::BinaryOutputArchive ar_out("./dump.data");
+
+    // Add a new entry
+    mp1[100] = 99;
+    mp1[300] = 299;
+    mp1[101] = 992;
+    mp1[1300] = 2991;
+    mp1[1130] = 299;
+    mp1[2130] = 1299;
+    // Iterate and print
+    for (const auto& n : mp1)
+        std::cout << "key: " << n.first << ", value: " << n.second << "\n";
+ 
+    mp1.dump(ar_out);
+    parallel_flat_hash_map<uint64_t, uint32_t> mp2;
+    phmap::BinaryInputArchive ar_in("./dump.data");
+
+    mp2.load(ar_in);
+    for (const auto& n : mp2)
+        std::cout << "key: " << n.first << ", value: " << n.second << "\n";
+}
+
+int main()
+{
+    dump_load_uint64_uint32();
+    dump_load_parallel_flat_hash_map();
+    return 0;
+}
+

examples/serialize.cc

@@ -22,9 +22,9 @@ void showtime(const char *name, std::function<void ()> doit)
     doit();
     auto t2 = std::chrono::high_resolution_clock::now();
     auto elapsed = milliseconds<double>(t2 - t1).count();
-    printf("%s: %.3fms\n", name, (int)elapsed / 1000.0f);
+    printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f);
 }
-    
+
 int main()
 {
     using MapType = phmap::flat_hash_map<bitset<42>, int>;

parallel_hashmap/phmap.h

@@ -1534,6 +1534,12 @@ public:
         }
     }
 
+    template<typename OutputArchive>
+    bool dump(OutputArchive&);
+
+    template<typename InputArchive>
+    bool load(InputArchive&);
+
     void rehash(size_t n) {
         if (n == 0 && capacity_ == 0) return;
         if (n == 0 && size_ == 0) {
@@ -3141,6 +3147,12 @@ public:
         a.swap(b);
     }
 
+    template<typename OutputArchive>
+    bool dump(OutputArchive& ar);
+
+    template<typename InputArchive>
+    bool load(InputArchive& ar);
+
 private:
     template <class Container, typename Enabler>
     friend struct phmap::container_internal::hashtable_debug_internal::HashtableDebugAccess;

parallel_hashmap/phmap_dump.h

@@ -0,0 +1,288 @@
+#if !defined(phmap_dump_h_guard_)
+#define phmap_dump_h_guard_
+
+// ---------------------------------------------------------------------------
+// Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com
+//
+//       providing dump/load/mmap_load
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ---------------------------------------------------------------------------
+
+#include <cstdint>
+#include <functional>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include "phmap.h"
+namespace phmap
+{
+
+namespace type_traits_internal {
+
+#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801
+    template<typename T> struct IsTriviallyCopyable : public std::integral_constant<bool, __has_trivial_copy(T)> {};
+#else
+    template<typename T> struct IsTriviallyCopyable : public std::is_trivially_copyable<T> {};
+#endif
+
+template <class T1, class T2>
+struct IsTriviallyCopyable<std::pair<T1, T2>> {
+    static constexpr bool value = IsTriviallyCopyable<T1>::value && IsTriviallyCopyable<T2>::value;
+};
+}
+
+namespace container_internal {
+
+//// raw_hash_set
+template <class Policy, class Hash, class Eq, class Alloc>
+template<typename OutputArchive>
+bool raw_hash_set<Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) {
+    static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
+                    "value_type should be dumpable");
+
+    typename OutputArchive::Guard guard(&ar);
+    if (!ar.dump(size_)) {
+        std::cerr << "Failed to dump size_" << std::endl;
+        return false;
+    }
+    if (size_ == 0) {
+        return true;
+    }
+    if (!ar.dump(capacity_)) {
+        std::cerr << "Failed to dump capacity_" << std::endl;
+        return false;
+    }
+    if (!ar.dump(reinterpret_cast<char*>(ctrl_),
+        sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) {
+
+        std::cerr << "Failed to dump ctrl_" << std::endl;
+        return false;
+    }
+    if (!ar.dump(reinterpret_cast<char*>(slots_),
+                    sizeof(slot_type) * capacity_)) {
+        std::cerr << "Failed to dump slot_" << std::endl;
+        return false;
+    }
+    return true;
+}
+
+template <class Policy, class Hash, class Eq, class Alloc>
+template<typename InputArchive>
+bool raw_hash_set<Policy, Hash, Eq, Alloc>::load(InputArchive& ar) {
+    static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
+                    "value_type should be dumpable");
+    
+    typename InputArchive::Guard guard(&ar);
+    if (!ar.load(&size_)) {
+        std::cerr << "Failed to load size_" << std::endl;
+        return false;
+    }
+    if (size_ == 0) {
+        return true;
+    }
+    if (!ar.load(&capacity_)) {
+        std::cerr << "Failed to load capacity_" << std::endl;
+        return false;
+    }
+
+    // allocate memory for ctrl_ and slots_
+    initialize_slots();
+    if (!ar.load(reinterpret_cast<char*>(ctrl_),
+        sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) {
+        std::cerr << "Failed to load ctrl" << std::endl;
+        return false;
+    }
+    if (!ar.load(reinterpret_cast<char*>(slots_),
+                    sizeof(slot_type) * capacity_)) {
+        std::cerr << "Failed to load slot" << std::endl;
+        return false;
+    }
+    return true;
+}
+
+////// parallel_hash_set
+template <size_t N,
+          template <class, class, class, class> class RefSet,
+          class Mtx_,
+          class Policy, class Hash, class Eq, class Alloc>
+template<typename OutputArchive>
+bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) {
+    static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
+                    "value_type should be dumpable");
+
+    typename OutputArchive::Guard guard(&ar);
+    if (! ar.dump(subcnt())) {
+        std::cerr << "Failed to dump meta!" << std::endl;
+        return false;
+    }
+    for (size_t i = 0; i < sets_.size(); ++i) {
+        auto& inner = sets_[i];
+        typename Lockable::UniqueLock m(const_cast<Inner&>(inner));
+        if (!inner.set_.dump(ar)) {
+            std::cerr << "Failed to dump submap " << i << std::endl;
+            return false;
+        }
+    }
+    return true;
+}
+
+template <size_t N,
+          template <class, class, class, class> class RefSet,
+          class Mtx_,
+          class Policy, class Hash, class Eq, class Alloc>
+template<typename InputArchive>
+bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::load(InputArchive& ar) {
+    static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
+                    "value_type should be dumpable");
+
+    typename InputArchive::Guard guard(&ar);
+    size_t submap_count = 0;
+    if (!ar.load(&submap_count)) {
+        std::cerr << "Failed to load submap count!" << std::endl;
+        return false;
+    }
+
+    if (submap_count != subcnt()) {
+        std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl;
+        return false;
+    }
+
+    for (size_t i = 0; i < submap_count; ++i) {            
+        auto& inner = sets_[i];
+        typename Lockable::UniqueLock m(const_cast<Inner&>(inner));
+        if (!inner.set_.load(ar)) {
+            std::cerr << "Failed to load submap " << i << std::endl;
+            return false;
+        }
+    }
+    return true;
+}
+} // namesapce container_internal
+
+
+
+// ArchiveOutput & ArchiveInput
+
+#define CHECK_FILE(f) {                                 \
+    if (!f.is_open()) {                                 \
+        std::cerr << "File is not open!" << std::endl;  \
+        return false;                                   \
+    }                                                   \
+}
+
+template<typename Archive>
+class ArchiveGuard {
+public:
+    ArchiveGuard(Archive* ar): ar_(ar) {
+        if (ar_->guard_ == NULL) {
+            ar_->guard_ = this;
+        }
+    };
+    ~ArchiveGuard() {
+        if (ar_ && ar_->guard_ == this) {
+            ar_->finish();
+        }
+    }
+private:
+    Archive* ar_;
+};
+
+class BinaryOutputArchive {
+public:
+    using Guard = ArchiveGuard<BinaryOutputArchive>;
+
+    BinaryOutputArchive(const std::string& file_path): offset_(0), guard_(NULL) {
+        ofs_.open(file_path.c_str(), std::ios_base::binary);
+    }
+
+    virtual ~BinaryOutputArchive() {
+        finish();
+    }
+
+    bool dump(char* p, size_t sz) {
+        CHECK_FILE(ofs_);
+        ofs_.write(p, sz);
+        offset_ += sz;
+        return true;
+    }
+
+    template<typename V>
+    typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type
+    dump(const V& v) {
+        CHECK_FILE(ofs_);        
+        ofs_.write(reinterpret_cast<char*>(const_cast<V*>(&v)), sizeof(V));
+        offset_ += sizeof(V);
+        return true;
+    }
+
+    void finish() {
+        if (ofs_.is_open()) {
+            ofs_.close();
+            offset_ = 0;
+        }
+    }
+
+private:
+    friend class ArchiveGuard<BinaryOutputArchive>;
+    std::ofstream ofs_;
+    size_t offset_;
+    Guard* guard_;
+};
+
+
+class BinaryInputArchive {
+public:
+    using Guard = ArchiveGuard<BinaryInputArchive>;
+
+    BinaryInputArchive(const std::string& file_path): offset_(0), guard_(NULL) {
+        ifs_.open(file_path.c_str(), std::ios_base::binary);
+    }
+
+    virtual ~BinaryInputArchive() {
+        finish();
+    }
+
+    bool load(char* p, size_t sz) {
+        CHECK_FILE(ifs_);
+        ifs_.read(p, sz);
+        offset_ += sz;
+        return true;
+    }
+
+    template<typename V>
+    typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type
+    load(V* v) {
+        CHECK_FILE(ifs_);
+        ifs_.read(reinterpret_cast<char*>(v), sizeof(V));
+        offset_ += sizeof(V);
+        return true;
+    }
+
+    void finish() {
+        if (ifs_.is_open()) {
+            ifs_.close();
+            offset_ = 0;
+        }
+    }
+   
+private:
+    friend class ArchiveGuard<BinaryInputArchive>;
+    std::ifstream ifs_;
+    size_t offset_;
+    Guard* guard_;
+};
+
+} // namespace phmap
+
+#endif // phmap_dump_h_guard_

tests/dump_load_test.cc

@@ -0,0 +1,82 @@
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "parallel_hashmap/phmap_dump.h"
+
+namespace phmap {
+namespace container_internal {
+namespace {
+
+using ::phmap::flat_hash_set;
+using ::phmap::flat_hash_map;
+using ::phmap::parallel_flat_hash_map;
+using ::phmap::BinaryOutputArchive;
+using ::phmap::BinaryInputArchive;
+
+TEST(DumpLoad, FlatHashSet_uin32) {
+    flat_hash_set<uint32_t> st1;
+    BinaryOutputArchive ar_out("./dump.data");
+
+    st1.insert(1991);
+    st1.insert(1202);
+
+    EXPECT_TRUE(st1.dump(ar_out));
+    flat_hash_set<uint32_t> st2;
+    BinaryInputArchive ar_in("./dump.data");
+
+    EXPECT_TRUE(st2.load(ar_in));
+
+    EXPECT_EQ(2, st2.size());
+    EXPECT_TRUE(st2.count(1991));
+    EXPECT_TRUE(st2.count(1202));
+}
+
+TEST(DumpLoad, FlatHashMap_uint64_uint32) {
+    flat_hash_map<uint64_t, uint32_t> mp1;
+    BinaryOutputArchive ar_out("./dump.data");
+
+    mp1[78731] = 99;
+    mp1[13141] = 299;
+    mp1[2651] = 101;
+
+    EXPECT_TRUE(mp1.dump(ar_out));
+    flat_hash_map<uint64_t, uint32_t> mp2;
+    BinaryInputArchive ar_in("./dump.data");
+
+    EXPECT_TRUE(mp2.load(ar_in));
+
+    EXPECT_EQ(3, mp2.size());
+    EXPECT_TRUE(mp2.count(78731));
+    EXPECT_TRUE(mp2.count(13141));
+    EXPECT_EQ(99, mp2.at(78731));
+    EXPECT_EQ(101, mp2.at(2651));
+}
+
+TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) {
+    parallel_flat_hash_map<uint64_t, uint32_t> mp1;
+    BinaryOutputArchive ar_out("./dump.data");
+
+    mp1[100] = 99;
+    mp1[300] = 299;
+    mp1[101] = 992;
+    mp1[1300] = 2991;
+    mp1[1130] = 299;
+    mp1[2130] = 1299;
+
+    EXPECT_TRUE(mp1.dump(ar_out));
+    parallel_flat_hash_map<uint64_t, uint32_t> mp2;
+    BinaryInputArchive ar_in("./dump.data");
+
+    EXPECT_TRUE(mp2.load(ar_in));
+    EXPECT_EQ(6, mp2.size());
+    EXPECT_EQ(99, mp2[100]);
+    EXPECT_EQ(299, mp2[300]);
+    EXPECT_EQ(299, mp2[1130]);
+    EXPECT_EQ(1299, mp2[2130]);
+}
+
+}
+}
+}
+