saberbud/parallel-hashmap
1
0
Fork 0
mirror of https://github.com/greg7mdp/parallel-hashmap.git synced 2026-08-29 08:34:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

use single file to store data for parallel hash

Browse Source
This commit is contained in:
sunkaicheng
2019-08-21 09:01:56 +08:00
parent e3c55f9830
commit 61f1bdb037
4 changed files with 41 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/dump_load.cc
+4 -4
View File
@@ -27,7 +27,7 @@ void dump_load_uint64_uint32() {
void dump_load_parallel_flat_hash_map() {
parallel_flat_hash_map<uint64_t, uint32_t> mp1;
phmap::OutputArchiveWrapper<phmap::BinaryOutputArchive> w_out("./");
phmap::BinaryOutputArchive ar_out("./dump.data");
// Add a new entry
mp1[100] = 99;
@@ -40,11 +40,11 @@ void dump_load_parallel_flat_hash_map() {
for (const auto& n : mp1)
std::cout << "key: " << n.first << ", value: " << n.second << "\n";
mp1.dump(w_out);
mp1.dump(ar_out);
parallel_flat_hash_map<uint64_t, uint32_t> mp2;
phmap::InputArchiveWrapper<phmap::BinaryInputArchive> w_in("./");
phmap::BinaryInputArchive ar_in("./dump.data");
mp2.load(w_in);
mp2.load(ar_in);
for (const auto& n : mp2)
std::cout << "key: " << n.first << ", value: " << n.second << "\n";
}
parallel_hashmap/phmap.h
Vendored
+21 -17
View File
@@ -1553,7 +1553,7 @@ public:
return false;
}
if (!ar.dump(reinterpret_cast<char*>(ctrl_),
sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) {
sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) {
std::cerr << "Failed to dump ctrl_" << std::endl;
return false;
@@ -1583,7 +1583,7 @@ public:
// allocate memory for ctrl_ and slots_
initialize_slots();
if (!ar.load(reinterpret_cast<char*>(ctrl_),
sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) {
sizeof(ctrl_t) * (capacity_ + Group::kWidth + 1))) {
std::cerr << "Failed to load ctrl" << std::endl;
return false;
}
@@ -3199,40 +3199,44 @@ public:
a.swap(b);
}
template<typename OutputArchiveWrapper, typename V = value_type>
template<typename OutputArchive, typename V = value_type>
typename std::enable_if<type_traits_internal::IsDumpableType<V>::value, bool>::type
dump(OutputArchiveWrapper& w) {
dump(OutputArchive& ar) {
typename OutputArchive::Guard guard(&ar);
if (! ar.dump(subcnt())) {
std::cerr << "Failed to dump meta!" << std::endl;
return false;
}
for (size_t i = 0; i < sets_.size(); ++i) {
auto& inner = sets_[i];
auto ar = w.create_archive(i);
typename Lockable::UniqueLock m(const_cast<Inner&>(inner));
if (!inner.set_.dump(*ar)) {
if (!inner.set_.dump(ar)) {
std::cerr << "Failed to dump submap " << i << std::endl;
return false;
}
}
if (! w.dump_meta(subcnt())) {
std::cerr << "Failed to dump meta!" << std::endl;
return false;
}
return true;
}
template<typename InputArchiveWrapper, typename V = value_type>
template<typename InputArchive, typename V = value_type>
typename std::enable_if<type_traits_internal::IsDumpableType<V>::value, bool>::type
load(InputArchiveWrapper& w) {
size_t submap_count = w.load_meta();
load(InputArchive& ar) {
typename InputArchive::Guard guard(&ar);
size_t submap_count = 0;
if (!ar.load(&submap_count)) {
std::cerr << "Failed to load submap count!" << std::endl;
return false;
}
if (submap_count != subcnt()) {
std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl;
return false;
}
for (size_t i = 0; i < sets_.size(); ++i) {
auto ar = w.create_archive(i);
for (size_t i = 0; i < submap_count; ++i) {
auto& inner = sets_[i];
if (!inner.set_.load(*ar)) {
typename Lockable::UniqueLock m(const_cast<Inner&>(inner));
if (!inner.set_.load(ar)) {
std::cerr << "Failed to load submap " << i << std::endl;
return false;
}
parallel_hashmap/phmap_utils.h
Vendored
+12 -58
View File
@@ -322,9 +322,13 @@ using HashState = HashStateBase<size_t>;
template<typename Archive>
class ArchiveGuard {
public:
ArchiveGuard(Archive* ar): ar_(ar) {};
ArchiveGuard(Archive* ar): ar_(ar) {
if (ar_->guard_ == NULL) {
ar_->guard_ = this;
}
};
~ArchiveGuard() {
if (ar_) {
if (ar_ && ar_->guard_ == this) {
ar_->finish();
}
}
@@ -336,7 +340,7 @@ class BinaryOutputArchive {
public:
using Guard = ArchiveGuard<BinaryOutputArchive>;
BinaryOutputArchive(const std::string& file_path) {
BinaryOutputArchive(const std::string& file_path): guard_(NULL) {
ofs_.open(file_path.c_str(), std::ios_base::binary);
}
@@ -364,7 +368,9 @@ public:
}
}
private:
friend class ArchiveGuard<BinaryOutputArchive>;
std::ofstream ofs_;
Guard* guard_;
};
@@ -372,7 +378,7 @@ class BinaryInputArchive {
public:
using Guard = ArchiveGuard<BinaryInputArchive>;
BinaryInputArchive(const std::string& file_path) {
BinaryInputArchive(const std::string& file_path): guard_(NULL) {
ifs_.open(file_path.c_str(), std::ios_base::binary);
}
@@ -400,62 +406,10 @@ public:
}
}
private:
friend class ArchiveGuard<BinaryInputArchive>;
std::ifstream ifs_;
Guard* guard_;
};
template<typename T = BinaryOutputArchive>
class OutputArchiveWrapper {
public:
using SubArchive = T;
OutputArchiveWrapper(const std::string& dir): dir_(dir) {
}
virtual ~OutputArchiveWrapper() {
}
bool dump_meta(size_t subcnt) {
auto ar = std::make_shared<SubArchive>(dir_ + "/meta.dump");
typename SubArchive::Guard guard(ar.get());
ar->dump(subcnt);
return true;
}
std::shared_ptr<SubArchive> create_archive(size_t i) {
std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump";
return std::make_shared<SubArchive>(file_path);
}
private:
std::string dir_;
};
template<typename T = BinaryInputArchive>
class InputArchiveWrapper {
public:
using SubArchive = T;
InputArchiveWrapper(const std::string& dir): dir_(dir) {
}
virtual ~InputArchiveWrapper() {
}
size_t load_meta() {
size_t subcnt = 0;
auto ar = std::make_shared<SubArchive>(dir_ + "/meta.dump");
typename SubArchive::Guard guard(ar.get());
ar->load(&subcnt);
return subcnt;
}
std::shared_ptr<SubArchive> create_archive(size_t i) {
std::string file_path = dir_ + "/sub_" + std::to_string(i) + ".dump";
return std::make_shared<SubArchive>(file_path);
}
private:
std::string dir_;
};
} // namespace phmap
tests/dump_load_test.cc
+4 -6
View File
@@ -14,8 +14,6 @@ using ::phmap::flat_hash_map;
using ::phmap::parallel_flat_hash_map;
using ::phmap::BinaryOutputArchive;
using ::phmap::BinaryInputArchive;
using ::phmap::OutputArchiveWrapper;
using ::phmap::InputArchiveWrapper;
TEST(DumpLoad, FlatHashSet_uin32) {
flat_hash_set<uint32_t> st1;
@@ -58,7 +56,7 @@ TEST(DumpLoad, FlatHashMap_uint64_uint32) {
TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) {
parallel_flat_hash_map<uint64_t, uint32_t> mp1;
OutputArchiveWrapper<BinaryOutputArchive> w_out("./");
BinaryOutputArchive ar_out("./dump.data");
mp1[100] = 99;
mp1[300] = 299;
@@ -67,11 +65,11 @@ TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) {
mp1[1130] = 299;
mp1[2130] = 1299;
EXPECT_TRUE(mp1.dump(w_out));
EXPECT_TRUE(mp1.dump(ar_out));
parallel_flat_hash_map<uint64_t, uint32_t> mp2;
InputArchiveWrapper<BinaryInputArchive> w_in("./");
BinaryInputArchive ar_in("./dump.data");
EXPECT_TRUE(mp2.load(w_in));
EXPECT_TRUE(mp2.load(ar_in));
EXPECT_EQ(6, mp2.size());
EXPECT_EQ(99, mp2[100]);