mirror of
https://github.com/greg7mdp/parallel-hashmap.git
synced 2026-08-29 08:34:39 +08:00
Add example mt_word_counter
This commit is contained in:
Vendored
+1
@@ -195,6 +195,7 @@ if (PHMAP_BUILD_EXAMPLES)
|
|||||||
add_executable(ex_dump_load examples/dump_load.cc phmap.natvis)
|
add_executable(ex_dump_load examples/dump_load.cc phmap.natvis)
|
||||||
add_executable(ex_btree examples/btree.cc phmap.natvis)
|
add_executable(ex_btree examples/btree.cc phmap.natvis)
|
||||||
add_executable(ex_matt examples/matt.cc phmap.natvis)
|
add_executable(ex_matt examples/matt.cc phmap.natvis)
|
||||||
|
add_executable(ex_mt_word_counter examples/mt_word_counter.cc phmap.natvis)
|
||||||
|
|
||||||
target_link_libraries(ex_knucleotide Threads::Threads)
|
target_link_libraries(ex_knucleotide Threads::Threads)
|
||||||
target_link_libraries(ex_bench Threads::Threads)
|
target_link_libraries(ex_bench Threads::Threads)
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
#include <iostream>
|
||||||
|
#include <fstream>
|
||||||
|
#include <sstream>
|
||||||
|
#include <parallel_hashmap/phmap.h>
|
||||||
|
#include <parallel_hashmap/btree.h>
|
||||||
|
#include <thread>
|
||||||
|
#include <array>
|
||||||
|
#include <vector>
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cstdlib>
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
* count the number of occurrences of each word in a large text file using multiple threads
|
||||||
|
*/
|
||||||
|
|
||||||
|
int main() {
|
||||||
|
// download Jane Austin "Pride and Prejudice"
|
||||||
|
// ------------------------------------------
|
||||||
|
if (system("wget https://www.gutenberg.org/files/1342/1342-0.txt") != 0) {
|
||||||
|
std::cout << "Error: could not retrieve test file https://www.gutenberg.org/files/1342/1342-0.txt\n";
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string filename = "1342-0.txt";
|
||||||
|
|
||||||
|
constexpr int num_threads = 4;
|
||||||
|
std::vector<std::thread> threads;
|
||||||
|
std::array<std::vector<std::string>, num_threads> lines_array;
|
||||||
|
|
||||||
|
{
|
||||||
|
// populate 4 vectors with lines from the book
|
||||||
|
std::ifstream file(filename);
|
||||||
|
if (!file.is_open()) {
|
||||||
|
std::cout << "Error: could not open file " << filename << std::endl;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
int line_idx = 0;
|
||||||
|
std::string line;
|
||||||
|
while (std::getline(file, line)) {
|
||||||
|
lines_array[line_idx % num_threads].push_back(std::move(line));
|
||||||
|
++line_idx;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
using Map = phmap::parallel_flat_hash_map_m<std::string, int>; // parallel_flat_hash_map_m has default internal mutex
|
||||||
|
Map word_counts;
|
||||||
|
|
||||||
|
// run 4 threads, each thread processing lines from one of the vectors
|
||||||
|
// -------------------------------------------------------------------
|
||||||
|
for (int i = 0; i < num_threads; ++i) {
|
||||||
|
threads.emplace_back(
|
||||||
|
[&word_counts](std::vector<std::string>&& lines) {
|
||||||
|
for (auto& line : lines) {
|
||||||
|
std::replace_if(line.begin(), line.end(), [](char c) -> bool { return !std::isalnum(c); }, ' ');
|
||||||
|
std::istringstream iss(line);
|
||||||
|
std::string word;
|
||||||
|
while (iss >> word) {
|
||||||
|
// use lazy_emplace to modify the map while the mutex is locked
|
||||||
|
word_counts.lazy_emplace_l(word,
|
||||||
|
[&](Map::value_type& p) { ++p.second; }, // called only when key was already present
|
||||||
|
[&](const Map::constructor& ctor) // construct value_type in place when key not present
|
||||||
|
{ ctor(std::move(word), 1); } );
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
std::move(lines_array[i]));
|
||||||
|
}
|
||||||
|
|
||||||
|
for (auto& thread : threads)
|
||||||
|
thread.join();
|
||||||
|
|
||||||
|
// print one word used at each frequency
|
||||||
|
// -------------------------------------
|
||||||
|
phmap::btree_map<int, std::string> result;
|
||||||
|
for (const auto& pair : word_counts)
|
||||||
|
result[pair.second] = pair.first;
|
||||||
|
|
||||||
|
for (const auto& p : result)
|
||||||
|
std::cout << p.first << ": " << p.second << std::endl;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
Vendored
+32
@@ -20,6 +20,7 @@
|
|||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
|
#include <mutex>
|
||||||
|
|
||||||
#if defined(PHMAP_USE_ABSL_HASH) && !defined(ABSL_HASH_HASH_H_)
|
#if defined(PHMAP_USE_ABSL_HASH) && !defined(ABSL_HASH_HASH_H_)
|
||||||
namespace absl { template <class T> struct Hash; };
|
namespace absl { template <class T> struct Hash; };
|
||||||
@@ -127,6 +128,37 @@ namespace phmap {
|
|||||||
class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks
|
class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks
|
||||||
class parallel_node_hash_map;
|
class parallel_node_hash_map;
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------------
|
||||||
|
// phmap::parallel_*_hash_* using std::mutex by default
|
||||||
|
// -----------------------------------------------------------------------------
|
||||||
|
template <class T,
|
||||||
|
class Hash = phmap::priv::hash_default_hash<T>,
|
||||||
|
class Eq = phmap::priv::hash_default_eq<T>,
|
||||||
|
class Alloc = phmap::priv::Allocator<T>,
|
||||||
|
size_t N = 4>
|
||||||
|
using parallel_flat_hash_set_m = parallel_flat_hash_set<T, Hash, Eq, Alloc, N, std::mutex>;
|
||||||
|
|
||||||
|
template <class K, class V,
|
||||||
|
class Hash = phmap::priv::hash_default_hash<K>,
|
||||||
|
class Eq = phmap::priv::hash_default_eq<K>,
|
||||||
|
class Alloc = phmap::priv::Allocator<phmap::priv::Pair<const K, V>>,
|
||||||
|
size_t N = 4>
|
||||||
|
using parallel_flat_hash_map_m = parallel_flat_hash_map<K, V, Hash, Eq, Alloc, N, std::mutex>;
|
||||||
|
|
||||||
|
template <class T,
|
||||||
|
class Hash = phmap::priv::hash_default_hash<T>,
|
||||||
|
class Eq = phmap::priv::hash_default_eq<T>,
|
||||||
|
class Alloc = phmap::priv::Allocator<T>,
|
||||||
|
size_t N = 4>
|
||||||
|
using parallel_node_hash_set_m = parallel_node_hash_set<T, Hash, Eq, Alloc, N, std::mutex>;
|
||||||
|
|
||||||
|
template <class K, class V,
|
||||||
|
class Hash = phmap::priv::hash_default_hash<K>,
|
||||||
|
class Eq = phmap::priv::hash_default_eq<K>,
|
||||||
|
class Alloc = phmap::priv::Allocator<phmap::priv::Pair<const K, V>>,
|
||||||
|
size_t N = 4>
|
||||||
|
using parallel_node_hash_map_m = parallel_node_hash_map<K, V, Hash, Eq, Alloc, N, std::mutex>;
|
||||||
|
|
||||||
// ------------- forward declarations for btree containers ----------------------------------
|
// ------------- forward declarations for btree containers ----------------------------------
|
||||||
template <typename Key, typename Compare = phmap::Less<Key>,
|
template <typename Key, typename Compare = phmap::Less<Key>,
|
||||||
typename Alloc = phmap::Allocator<Key>>
|
typename Alloc = phmap::Allocator<Key>>
|
||||||
|
|||||||
Reference in New Issue
Block a user