saberbud/parallel-hashmap
1
0
Fork 0
mirror of https://github.com/greg7mdp/parallel-hashmap.git synced 2026-08-29 08:34:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

added serialize compare and added parallel dump&load

Browse Source
This commit is contained in:
sunkaicheng
2019-08-15 12:23:05 +08:00
parent da55a69dfa
commit 5cd20dc4a6
5 changed files with 233 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/dump_load.cc
+36 -12
View File
@@ -4,8 +4,9 @@
using phmap::flat_hash_map;
using phmap::flat_hash_set;
using phmap::parallel_flat_hash_map;
void load_dump_string_string() {
void dump_load_string_string() {
flat_hash_map<std::string, std::string> mp1;
// Add a new entry
@@ -27,7 +28,7 @@ void load_dump_string_string() {
std::remove("./dump.data");
}
void load_dump_uint64_uint32() {
void dump_load_uint64_uint32() {
flat_hash_map<uint64_t, uint32_t> mp1;
// Add a new entry
@@ -49,7 +50,7 @@ void load_dump_uint64_uint32() {
std::remove("./dump.data");
}
void load_dump_string_uint32() {
void dump_load_string_uint32() {
flat_hash_map<std::string, uint32_t> mp1;
// Add a new entry
@@ -71,7 +72,7 @@ void load_dump_string_uint32() {
std::remove("./dump.data");
}
void load_dump_uint32_string() {
void dump_load_uint32_string() {
flat_hash_map<uint32_t, std::string> mp1;
// Add a new entry
@@ -93,7 +94,7 @@ void load_dump_uint32_string() {
std::remove("./dump.data");
}
void load_dump_string() {
void dump_load_string() {
flat_hash_set<std::string> st1;
// Add a new entry
@@ -115,7 +116,7 @@ void load_dump_string() {
std::remove("./dump.data");
}
void load_dump_uint64() {
void dump_load_uint64() {
flat_hash_set<uint64_t> st1;
// Add a new entry
@@ -137,13 +138,36 @@ void load_dump_uint64() {
std::remove("./dump.data");
}
void dump_load_parallel_flat_hash_map() {
parallel_flat_hash_map<uint64_t, uint32_t> mp1;
// Add a new entry
mp1[100] = 99;
mp1[300] = 299;
mp1[101] = 992;
mp1[1300] = 2991;
mp1[1130] = 299;
mp1[2130] = 1299;
// Iterate and print
for (const auto& n : mp1)
std::cout << "key: " << n.first << ", value: " << n.second << "\n";
mp1.dump("./dump");
parallel_flat_hash_map<uint64_t, uint32_t> mp2;
mp2.load("./dump");
for (const auto& n : mp2)
std::cout << "key: " << n.first << ", value: " << n.second << "\n";
}
int main()
{
load_dump_string_string();
load_dump_uint64_uint32();
load_dump_string_uint32();
load_dump_uint32_string();
load_dump_string();
load_dump_uint64();
dump_load_string_string();
dump_load_uint64_uint32();
dump_load_string_uint32();
dump_load_uint32_string();
dump_load_string();
dump_load_uint64();
dump_load_parallel_flat_hash_map();
return 0;
}
examples/serialize.cc
+2 -2
View File
@@ -22,9 +22,9 @@ void showtime(const char *name, std::function<void ()> doit)
doit();
auto t2 = std::chrono::high_resolution_clock::now();
auto elapsed = milliseconds<double>(t2 - t1).count();
printf("%s: %.3fms\n", name, (int)elapsed / 1000.0f);
printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f);
}
int main()
{
using MapType = phmap::flat_hash_map<bitset<42>, int>;
examples/serialize_compare.cc
+130
View File
@@ -0,0 +1,130 @@
#include <iostream>
#include <bitset>
#include <cinttypes>
#include "parallel_hashmap/phmap.h"
#include "cereal/types/unordered_map.hpp"
#include "cereal/types/memory.hpp"
#include "cereal/types/bitset.hpp"
#include "cereal/archives/binary.hpp"
#include <fstream>
#include <random>
#include <chrono>
#include <functional>
#include <cstdio>
using phmap::flat_hash_map;
using namespace std;
template <typename T> using milliseconds = std::chrono::duration<T, std::milli>;
void showtime(const char *name, std::function<void ()> doit)
{
auto t1 = std::chrono::high_resolution_clock::now();
doit();
auto t2 = std::chrono::high_resolution_clock::now();
auto elapsed = milliseconds<double>(t2 - t1).count();
printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f);
}
const size_t bigger_than_cachesize = 10 * 1024 * 1024;
long *p = new long[bigger_than_cachesize];
long *p1 = new long[bigger_than_cachesize];
int main(int argc, char* argv[])
{
size_t num_items = 2000000;
if (argc == 2) {
num_items = atoi(argv[1]);
}
std::cout << "items size: " << num_items << std::endl;
using MapType = phmap::flat_hash_map<uint64_t, int>;
MapType table;
std::vector<uint64_t> test_data_arr(num_items, 0);
for (size_t i = 0; i < num_items; i++) {
test_data_arr[i] = i;
}
std::srand(std::time(nullptr));
auto generate_random = [](int n) -> size_t {
return std::rand() % n;
};
auto shuffle = [&generate_random](std::vector<uint64_t>& arr) {
size_t n = arr.size();
size_t pos = n;
while(pos > 0) {
size_t r = generate_random(pos);
swap(arr[r], arr[pos - 1]);
pos --;
}
};
// shuffle
shuffle(test_data_arr);
// Iterate and add keys and values
// -------------------------------
showtime("build hash", [&table, &test_data_arr, num_items]() {
table.reserve(num_items);
for (int i=0; i < num_items; ++i) {
table[test_data_arr[i]] = rand();
}
});
// cerealize and save data
// -----------------------
showtime("serialize", [&table]() {
ofstream os("out.cereal", ios::binary);
cereal::BinaryOutputArchive archive(os);
archive(table.size());
archive(table);
});
MapType().swap(table); // make sure table is newly created
// "flush" cache.
for(int i = 0; i < bigger_than_cachesize; i++) {
p[i] = rand();
}
// deserialize
// -----------
showtime("deserialize", [&table]() {
ifstream is("out.cereal", ios::binary);
cereal::BinaryInputArchive archive_in(is);
size_t table_size;
archive_in(table_size);
table.reserve(table_size);
archive_in(table); // deserialize from file out.cereal into variable
});
printf("table size: %zu\n", table.size());
// dump data
// -----------------------
showtime("dump", [&table]() {
table.dump("./out.dump");
});
// "flush" cache.
for(int i = 0; i < bigger_than_cachesize; i++) {
p1[i] = rand();
}
MapType().swap(table); // make sure table is newly created
// load
// -----------
showtime("load", [&table]() {
table.load("./out.dump");
});
printf("table size: %zu\n", table.size());
return 0;
}