saberbud/parallel-hashmap
1
0
Fork 0
mirror of https://github.com/greg7mdp/parallel-hashmap.git synced 2026-08-29 08:34:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

added tests

Browse Source
This commit is contained in:
sunkaicheng
2019-08-18 22:50:47 +08:00
parent 18614d316d
commit ba121d043f
5 changed files with 116 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
Vendored
+4 -1
View File
@@ -111,6 +111,10 @@ if (PHMAP_BUILD_TESTS)
phmap_cc_test(NAME parallel_flat_hash_map_mutex SRCS "tests/parallel_flat_hash_map_mutex_test.cc"
COPTS "-DUNORDERED_MAP_CXX17" DEPS gmock_main)
phmap_cc_test(NAME dump_load SRCS "tests/dump_load_test.cc"
COPTS "-DUNORDERED_MAP_CXX17" DEPS gmock_main)
endif()
if (PHMAP_BUILD_EXAMPLES)
@@ -132,7 +136,6 @@ if (PHMAP_BUILD_EXAMPLES)
add_executable(ex_insert_bench examples/insert_bench.cc phmap.natvis)
add_executable(ex_knucleotide examples/knucleotide.cc phmap.natvis)
add_executable(ex_dump_load examples/dump_load.cc phmap.natvis)
# add_executable(ex_serialize_compare examples/serialize_compare.cc phmap.natvis)
target_link_libraries(ex_knucleotide Threads::Threads)
target_link_libraries(ex_bench Threads::Threads)
examples/dump_load.cc
+1 -128
View File
@@ -3,33 +3,8 @@
#include <parallel_hashmap/phmap.h>
using phmap::flat_hash_map;
using phmap::flat_hash_set;
using phmap::parallel_flat_hash_map;
void dump_load_string_string() {
flat_hash_map<std::string, std::string> mp1;
phmap::BinaryOutputArchive ar_out("./dump.data");
// Add a new entry
mp1["key-1"] = "value-1";
mp1["key-2"] = "value-2";
// Iterate and print keys and values
for (const auto& n : mp1)
std::cout << n.first << "'s value is: " << n.second << "\n";
mp1.dump(ar_out);
flat_hash_map<std::string, std::string> mp2;
phmap::BinaryInputArchive ar_in("./dump.data");
mp2.load(ar_in);
// Iterate and print keys and values g|++
for (const auto& n : mp2)
std::cout << n.first << "'s value is: " << n.second << "\n";
std::remove("./dump.data");
}
void dump_load_uint64_uint32() {
flat_hash_map<uint64_t, uint32_t> mp1;
phmap::BinaryOutputArchive ar_out("./dump.data");
@@ -48,103 +23,6 @@ void dump_load_uint64_uint32() {
// Iterate and print keys and values g|++
for (const auto& n : mp2)
std::cout << n.first << "'s value is: " << n.second << "\n";
std::remove("./dump.data");
}
void dump_load_string_uint32() {
flat_hash_map<std::string, uint32_t> mp1;
phmap::BinaryOutputArchive ar_out("./dump.data");
// Add a new entry
mp1["key-1"] = 99;
mp1["key-2"] = 299;
// Iterate and print keys and values
for (const auto& n : mp1)
std::cout << n.first << "'s value is: " << n.second << "\n";
mp1.dump(ar_out);
flat_hash_map<std::string, uint32_t> mp2;
phmap::BinaryInputArchive ar_in("./dump.data");
mp2.load(ar_in);
// Iterate and print keys and values g|++
for (const auto& n : mp2)
std::cout << n.first << "'s value is: " << n.second << "\n";
std::remove("./dump.data");
}
void dump_load_uint32_string() {
flat_hash_map<uint32_t, std::string> mp1;
phmap::BinaryOutputArchive ar_out("./dump.data");
// Add a new entry
mp1[100] = "hello";
mp1[299] = "world";
// Iterate and print keys and values
for (const auto& n : mp1)
std::cout << n.first << "'s value is: " << n.second << "\n";
mp1.dump(ar_out);
flat_hash_map<uint32_t, std::string> mp2;
phmap::BinaryInputArchive ar_in("./dump.data");
mp2.load(ar_in);
// Iterate and print keys and values g|++
for (const auto& n : mp2)
std::cout << n.first << "'s value is: " << n.second << "\n";
std::remove("./dump.data");
}
void dump_load_string() {
flat_hash_set<std::string> st1;
phmap::BinaryOutputArchive ar_out("./dump.data");
// Add a new entry
st1.insert("hello");
st1.insert("world");
// Iterate and print
for (const auto& n : st1)
std::cout << "value: " << n << "\n";
st1.dump(ar_out);
flat_hash_set<std::string> st2;
phmap::BinaryInputArchive ar_in("./dump.data");
st2.load(ar_in);
// Iterate and print keys and values g|++
for (const auto& n : st2)
std::cout << "value: " << n << "\n";
std::remove("./dump.data");
}
void dump_load_uint64() {
flat_hash_set<uint64_t> st1;
phmap::BinaryOutputArchive ar_out("./dump.data");
// Add a new entry
st1.insert(878);
st1.insert(1424);
// Iterate and print
for (const auto& n : st1)
std::cout << "value: " << n << "\n";
st1.dump(ar_out);
flat_hash_set<uint64_t> st2;
phmap::BinaryInputArchive ar_in("./dump.data");
st2.load(ar_in);
// Iterate and print keys and values g|++
for (const auto& n : st2)
std::cout << "value: " << n << "\n";
std::remove("./dump.data");
}
void dump_load_parallel_flat_hash_map() {
@@ -172,13 +50,8 @@ void dump_load_parallel_flat_hash_map() {
}
int main()
{
dump_load_string_string();
{
dump_load_uint64_uint32();
dump_load_string_uint32();
dump_load_uint32_string();
dump_load_string();
dump_load_uint64();
dump_load_parallel_flat_hash_map();
return 0;
}
examples/serialize_compare.cc
-130
View File
@@ -1,130 +0,0 @@
#include <iostream>
#include <bitset>
#include <cinttypes>
#include "parallel_hashmap/phmap.h"
#include "cereal/types/unordered_map.hpp"
#include "cereal/types/memory.hpp"
#include "cereal/types/bitset.hpp"
#include "cereal/archives/binary.hpp"
#include <fstream>
#include <random>
#include <chrono>
#include <functional>
#include <cstdio>
using phmap::flat_hash_map;
using namespace std;
template <typename T> using milliseconds = std::chrono::duration<T, std::milli>;
void showtime(const char *name, std::function<void ()> doit)
{
auto t1 = std::chrono::high_resolution_clock::now();
doit();
auto t2 = std::chrono::high_resolution_clock::now();
auto elapsed = milliseconds<double>(t2 - t1).count();
printf("%s: %.3fs\n", name, (int)elapsed / 1000.0f);
}
const size_t bigger_than_cachesize = 10 * 1024 * 1024;
long *p = new long[bigger_than_cachesize];
long *p1 = new long[bigger_than_cachesize];
int main(int argc, char* argv[])
{
size_t num_items = 2000000;
if (argc == 2) {
num_items = atoi(argv[1]);
}
std::cout << "items size: " << num_items << std::endl;
using MapType = phmap::flat_hash_map<uint64_t, int>;
MapType table;
std::vector<uint64_t> test_data_arr(num_items, 0);
for (size_t i = 0; i < num_items; i++) {
test_data_arr[i] = i;
}
std::srand(std::time(nullptr));
auto generate_random = [](int n) -> size_t {
return std::rand() % n;
};
auto shuffle = [&generate_random](std::vector<uint64_t>& arr) {
size_t n = arr.size();
size_t pos = n;
while(pos > 0) {
size_t r = generate_random(pos);
swap(arr[r], arr[pos - 1]);
pos --;
}
};
// shuffle
shuffle(test_data_arr);
// Iterate and add keys and values
// -------------------------------
showtime("build hash", [&table, &test_data_arr, num_items]() {
table.reserve(num_items);
for (int i=0; i < num_items; ++i) {
table[test_data_arr[i]] = rand();
}
});
// cerealize and save data
// -----------------------
showtime("serialize", [&table]() {
ofstream os("out.cereal", ios::binary);
cereal::BinaryOutputArchive archive(os);
archive(table.size());
archive(table);
});
MapType().swap(table); // make sure table is newly created
// "flush" cache.
for(int i = 0; i < bigger_than_cachesize; i++) {
p[i] = rand();
}
// deserialize
// -----------
showtime("deserialize", [&table]() {
ifstream is("out.cereal", ios::binary);
cereal::BinaryInputArchive archive_in(is);
size_t table_size;
archive_in(table_size);
table.reserve(table_size);
archive_in(table); // deserialize from file out.cereal into variable
});
printf("table size: %zu\n", table.size());
// dump data
// -----------------------
showtime("dump", [&table]() {
table.dump("./out.dump");
});
// "flush" cache.
for(int i = 0; i < bigger_than_cachesize; i++) {
p1[i] = rand();
}
MapType().swap(table); // make sure table is newly created
// load
// -----------
showtime("load", [&table]() {
table.load("./out.dump");
});
printf("table size: %zu\n", table.size());
return 0;
}
parallel_hashmap/phmap.h
Vendored
+6 -3
View File
@@ -1552,14 +1552,16 @@ public:
std::cerr << "Failed to dump capacity_" << std::endl;
return false;
}
if (!ar.dump(reinterpret_cast<char*>(ctrl_), sizeof(ctrl_t) * capacity_)) {
if (!ar.dump(reinterpret_cast<char*>(ctrl_),
sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) {
std::cerr << "Failed to dump ctrl_" << std::endl;
return false;
}
if (!ar.dump(reinterpret_cast<char*>(slots_), sizeof(slot_type) * capacity_)) {
std::cerr << "Failed to dump slot_" << std::endl;
return false;
}
}
return true;
}
@@ -1580,7 +1582,8 @@ public:
}
// allocate memory for ctrl_ and slots_
initialize_slots();
if (!ar.load(reinterpret_cast<char*>(ctrl_), sizeof(ctrl_t) * capacity_)) {
if (!ar.load(reinterpret_cast<char*>(ctrl_),
sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) {
std::cerr << "Failed to load ctrl" << std::endl;
return false;
}
tests/dump_load_test.cc
+105
View File
@@ -0,0 +1,105 @@
#include <vector>
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "parallel_hashmap/phmap.h"
namespace phmap {
namespace container_internal {
namespace {
using ::phmap::flat_hash_set;
using ::phmap::flat_hash_map;
using ::phmap::parallel_flat_hash_map;
using ::phmap::BinaryOutputArchive;
using ::phmap::BinaryInputArchive;
using ::phmap::OutputArchiveWrapper;
using ::phmap::InputArchiveWrapper;
TEST(DumpLoad, FlatHashSet_string) {
flat_hash_set<std::string> st1;
BinaryOutputArchive ar_out("./dump.data");
st1.insert("hello");
st1.insert("world");
EXPECT_TRUE(st1.dump(ar_out));
flat_hash_set<std::string> st2;
BinaryInputArchive ar_in("./dump.data");
EXPECT_TRUE(st2.load(ar_in));
EXPECT_EQ(2, st2.size());
EXPECT_TRUE(st2.count("hello"));
EXPECT_TRUE(st2.count("world"));
}
TEST(DumpLoad, FlatHashMap_string_uint32) {
flat_hash_map<std::string, uint32_t> mp1;
BinaryOutputArchive ar_out("./dump.data");
mp1["key-1"] = 99;
mp1["key-2"] = 299;
EXPECT_TRUE(mp1.dump(ar_out));
flat_hash_map<std::string, uint32_t> mp2;
BinaryInputArchive ar_in("./dump.data");
EXPECT_TRUE(mp2.load(ar_in));
EXPECT_EQ(2, mp2.size());
EXPECT_TRUE(mp2.count("key-1"));
EXPECT_TRUE(mp2.count("key-2"));
EXPECT_EQ(99, mp2.at("key-1"));
EXPECT_EQ(299, mp2.at("key-2"));
}
TEST(DumpLoad, FlatHashMap_uint64_uint32) {
flat_hash_map<uint64_t, uint32_t> mp1;
BinaryOutputArchive ar_out("./dump.data");
mp1[78731] = 99;
mp1[13141] = 299;
mp1[2651] = 101;
EXPECT_TRUE(mp1.dump(ar_out));
flat_hash_map<uint64_t, uint32_t> mp2;
BinaryInputArchive ar_in("./dump.data");
EXPECT_TRUE(mp2.load(ar_in));
EXPECT_EQ(3, mp2.size());
EXPECT_TRUE(mp2.count(78731));
EXPECT_TRUE(mp2.count(13141));
EXPECT_EQ(99, mp2.at(78731));
EXPECT_EQ(101, mp2.at(2651));
}
TEST(DumpLoad, ParallelFlatHashMap_uint64_uint32) {
parallel_flat_hash_map<uint64_t, uint32_t> mp1;
OutputArchiveWrapper<BinaryOutputArchive> w_out("./");
mp1[100] = 99;
mp1[300] = 299;
mp1[101] = 992;
mp1[1300] = 2991;
mp1[1130] = 299;
mp1[2130] = 1299;
EXPECT_TRUE(mp1.dump(w_out));
parallel_flat_hash_map<uint64_t, uint32_t> mp2;
InputArchiveWrapper<BinaryInputArchive> w_in("./");
EXPECT_TRUE(mp2.load(w_in));
EXPECT_EQ(6, mp2.size());
EXPECT_EQ(99, mp2[100]);
EXPECT_EQ(299, mp2[300]);
EXPECT_EQ(299, mp2[1130]);
EXPECT_EQ(1299, mp2[2130]);
}
}
}
}