saberbud/parallel-hashmap
1
0
Fork 0
mirror of https://github.com/greg7mdp/parallel-hashmap.git synced 2026-08-30 09:00:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

simplify load/dump code. Also don't close the archibe after dump/load, we may need to save multiple hash maps in one archive.

Browse Source
This commit is contained in:
greg
2019-08-25 14:55:46 -04:00
parent 2ff7c0fce7
commit 77ef910921
2 changed files with 26 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
parallel_hashmap/phmap_dump.h
Vendored
+25 -86
View File
@@ -19,8 +19,6 @@
// limitations under the License.
// ---------------------------------------------------------------------------
#include <cstdint>
#include <functional>
#include <iostream>
#include <fstream>
#include <sstream>
@@ -44,14 +42,15 @@ struct IsTriviallyCopyable<std::pair<T1, T2>> {
namespace container_internal {
//// raw_hash_set
// ------------------------------------------------------------------------
// dump/load for raw_hash_set
// ------------------------------------------------------------------------
template <class Policy, class Hash, class Eq, class Alloc>
template<typename OutputArchive>
bool raw_hash_set<Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) {
static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
"value_type should be dumpable");
"value_type should be trivially copyable");
typename OutputArchive::Guard guard(&ar);
if (!ar.dump(size_)) {
std::cerr << "Failed to dump size_" << std::endl;
return false;
@@ -81,9 +80,8 @@ template <class Policy, class Hash, class Eq, class Alloc>
template<typename InputArchive>
bool raw_hash_set<Policy, Hash, Eq, Alloc>::load(InputArchive& ar) {
static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
"value_type should be dumpable");
typename InputArchive::Guard guard(&ar);
"value_type should be trivially copyable");
raw_hash_set<Policy, Hash, Eq, Alloc>().swap(*this); // clear any existing content
if (!ar.load(&size_)) {
std::cerr << "Failed to load size_" << std::endl;
return false;
@@ -111,7 +109,9 @@ bool raw_hash_set<Policy, Hash, Eq, Alloc>::load(InputArchive& ar) {
return true;
}
////// parallel_hash_set
// ------------------------------------------------------------------------
// dump/load for parallel_hash_set
// ------------------------------------------------------------------------
template <size_t N,
template <class, class, class, class> class RefSet,
class Mtx_,
@@ -119,9 +119,8 @@ template <size_t N,
template<typename OutputArchive>
bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::dump(OutputArchive& ar) {
static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
"value_type should be dumpable");
"value_type should be trivially copyable");
typename OutputArchive::Guard guard(&ar);
if (! ar.dump(subcnt())) {
std::cerr << "Failed to dump meta!" << std::endl;
return false;
@@ -144,9 +143,8 @@ template <size_t N,
template<typename InputArchive>
bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::load(InputArchive& ar) {
static_assert(type_traits_internal::IsTriviallyCopyable<value_type>::value,
"value_type should be dumpable");
"value_type should be trivially copyable");
typename InputArchive::Guard guard(&ar);
size_t submap_count = 0;
if (!ar.load(&submap_count)) {
std::cerr << "Failed to load submap count!" << std::endl;
@@ -168,119 +166,60 @@ bool parallel_hash_set<N, RefSet, Mtx_, Policy, Hash, Eq, Alloc>::load(InputArch
}
return true;
}
} // namesapce container_internal
} // namespace container_internal
// ArchiveOutput & ArchiveInput
#define CHECK_FILE(f) { \
if (!f.is_open()) { \
std::cerr << "File is not open!" << std::endl; \
return false; \
} \
}
template<typename Archive>
class ArchiveGuard {
public:
ArchiveGuard(Archive* ar): ar_(ar) {
if (ar_->guard_ == NULL) {
ar_->guard_ = this;
}
};
~ArchiveGuard() {
if (ar_ && ar_->guard_ == this) {
ar_->finish();
}
}
private:
Archive* ar_;
};
// ------------------------------------------------------------------------
// BinaryArchive
// File is closed when archive object is destroyed
// ------------------------------------------------------------------------
// ------------------------------------------------------------------------
// ------------------------------------------------------------------------
class BinaryOutputArchive {
public:
using Guard = ArchiveGuard<BinaryOutputArchive>;
BinaryOutputArchive(const std::string& file_path): offset_(0), guard_(NULL) {
ofs_.open(file_path.c_str(), std::ios_base::binary);
BinaryOutputArchive(const char *file_path) {
ofs_.open(file_path, std::ios_base::binary);
}
virtual ~BinaryOutputArchive() {
finish();
}
bool dump(char* p, size_t sz) {
CHECK_FILE(ofs_);
bool dump(const char *p, size_t sz) {
ofs_.write(p, sz);
offset_ += sz;
return true;
}
template<typename V>
typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type
dump(const V& v) {
CHECK_FILE(ofs_);
ofs_.write(reinterpret_cast<char*>(const_cast<V*>(&v)), sizeof(V));
offset_ += sizeof(V);
ofs_.write(reinterpret_cast<const char *>(&v), sizeof(V));
return true;
}
void finish() {
if (ofs_.is_open()) {
ofs_.close();
offset_ = 0;
}
}
private:
friend class ArchiveGuard<BinaryOutputArchive>;
std::ofstream ofs_;
size_t offset_;
Guard* guard_;
};
class BinaryInputArchive {
public:
using Guard = ArchiveGuard<BinaryInputArchive>;
BinaryInputArchive(const std::string& file_path): offset_(0), guard_(NULL) {
ifs_.open(file_path.c_str(), std::ios_base::binary);
}
virtual ~BinaryInputArchive() {
finish();
BinaryInputArchive(const char * file_path) {
ifs_.open(file_path, std::ios_base::binary);
}
bool load(char* p, size_t sz) {
CHECK_FILE(ifs_);
ifs_.read(p, sz);
offset_ += sz;
return true;
}
template<typename V>
typename std::enable_if<type_traits_internal::IsTriviallyCopyable<V>::value, bool>::type
load(V* v) {
CHECK_FILE(ifs_);
ifs_.read(reinterpret_cast<char*>(v), sizeof(V));
offset_ += sizeof(V);
ifs_.read(reinterpret_cast<char *>(v), sizeof(V));
return true;
}
void finish() {
if (ifs_.is_open()) {
ifs_.close();
offset_ = 0;
}
}
private:
friend class ArchiveGuard<BinaryInputArchive>;
std::ifstream ifs_;
size_t offset_;
Guard* guard_;
};
} // namespace phmap