mirror of
https://github.com/greg7mdp/parallel-hashmap.git
synced 2026-08-29 08:34:39 +08:00
Add example mt_word_counter
This commit is contained in:
Vendored
+1
@@ -195,6 +195,7 @@ if (PHMAP_BUILD_EXAMPLES)
|
||||
add_executable(ex_dump_load examples/dump_load.cc phmap.natvis)
|
||||
add_executable(ex_btree examples/btree.cc phmap.natvis)
|
||||
add_executable(ex_matt examples/matt.cc phmap.natvis)
|
||||
add_executable(ex_mt_word_counter examples/mt_word_counter.cc phmap.natvis)
|
||||
|
||||
target_link_libraries(ex_knucleotide Threads::Threads)
|
||||
target_link_libraries(ex_bench Threads::Threads)
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <parallel_hashmap/phmap.h>
|
||||
#include <parallel_hashmap/btree.h>
|
||||
#include <thread>
|
||||
#include <array>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
|
||||
|
||||
/*
|
||||
* count the number of occurrences of each word in a large text file using multiple threads
|
||||
*/
|
||||
|
||||
int main() {
|
||||
// download Jane Austin "Pride and Prejudice"
|
||||
// ------------------------------------------
|
||||
if (system("wget https://www.gutenberg.org/files/1342/1342-0.txt") != 0) {
|
||||
std::cout << "Error: could not retrieve test file https://www.gutenberg.org/files/1342/1342-0.txt\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
const std::string filename = "1342-0.txt";
|
||||
|
||||
constexpr int num_threads = 4;
|
||||
std::vector<std::thread> threads;
|
||||
std::array<std::vector<std::string>, num_threads> lines_array;
|
||||
|
||||
{
|
||||
// populate 4 vectors with lines from the book
|
||||
std::ifstream file(filename);
|
||||
if (!file.is_open()) {
|
||||
std::cout << "Error: could not open file " << filename << std::endl;
|
||||
return 1;
|
||||
}
|
||||
int line_idx = 0;
|
||||
std::string line;
|
||||
while (std::getline(file, line)) {
|
||||
lines_array[line_idx % num_threads].push_back(std::move(line));
|
||||
++line_idx;
|
||||
}
|
||||
}
|
||||
|
||||
using Map = phmap::parallel_flat_hash_map_m<std::string, int>; // parallel_flat_hash_map_m has default internal mutex
|
||||
Map word_counts;
|
||||
|
||||
// run 4 threads, each thread processing lines from one of the vectors
|
||||
// -------------------------------------------------------------------
|
||||
for (int i = 0; i < num_threads; ++i) {
|
||||
threads.emplace_back(
|
||||
[&word_counts](std::vector<std::string>&& lines) {
|
||||
for (auto& line : lines) {
|
||||
std::replace_if(line.begin(), line.end(), [](char c) -> bool { return !std::isalnum(c); }, ' ');
|
||||
std::istringstream iss(line);
|
||||
std::string word;
|
||||
while (iss >> word) {
|
||||
// use lazy_emplace to modify the map while the mutex is locked
|
||||
word_counts.lazy_emplace_l(word,
|
||||
[&](Map::value_type& p) { ++p.second; }, // called only when key was already present
|
||||
[&](const Map::constructor& ctor) // construct value_type in place when key not present
|
||||
{ ctor(std::move(word), 1); } );
|
||||
}
|
||||
}
|
||||
},
|
||||
std::move(lines_array[i]));
|
||||
}
|
||||
|
||||
for (auto& thread : threads)
|
||||
thread.join();
|
||||
|
||||
// print one word used at each frequency
|
||||
// -------------------------------------
|
||||
phmap::btree_map<int, std::string> result;
|
||||
for (const auto& pair : word_counts)
|
||||
result[pair.second] = pair.first;
|
||||
|
||||
for (const auto& p : result)
|
||||
std::cout << p.first << ": " << p.second << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
Vendored
+32
@@ -20,6 +20,7 @@
|
||||
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <mutex>
|
||||
|
||||
#if defined(PHMAP_USE_ABSL_HASH) && !defined(ABSL_HASH_HASH_H_)
|
||||
namespace absl { template <class T> struct Hash; };
|
||||
@@ -127,6 +128,37 @@ namespace phmap {
|
||||
class Mutex = phmap::NullMutex> // use std::mutex to enable internal locks
|
||||
class parallel_node_hash_map;
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// phmap::parallel_*_hash_* using std::mutex by default
|
||||
// -----------------------------------------------------------------------------
|
||||
template <class T,
|
||||
class Hash = phmap::priv::hash_default_hash<T>,
|
||||
class Eq = phmap::priv::hash_default_eq<T>,
|
||||
class Alloc = phmap::priv::Allocator<T>,
|
||||
size_t N = 4>
|
||||
using parallel_flat_hash_set_m = parallel_flat_hash_set<T, Hash, Eq, Alloc, N, std::mutex>;
|
||||
|
||||
template <class K, class V,
|
||||
class Hash = phmap::priv::hash_default_hash<K>,
|
||||
class Eq = phmap::priv::hash_default_eq<K>,
|
||||
class Alloc = phmap::priv::Allocator<phmap::priv::Pair<const K, V>>,
|
||||
size_t N = 4>
|
||||
using parallel_flat_hash_map_m = parallel_flat_hash_map<K, V, Hash, Eq, Alloc, N, std::mutex>;
|
||||
|
||||
template <class T,
|
||||
class Hash = phmap::priv::hash_default_hash<T>,
|
||||
class Eq = phmap::priv::hash_default_eq<T>,
|
||||
class Alloc = phmap::priv::Allocator<T>,
|
||||
size_t N = 4>
|
||||
using parallel_node_hash_set_m = parallel_node_hash_set<T, Hash, Eq, Alloc, N, std::mutex>;
|
||||
|
||||
template <class K, class V,
|
||||
class Hash = phmap::priv::hash_default_hash<K>,
|
||||
class Eq = phmap::priv::hash_default_eq<K>,
|
||||
class Alloc = phmap::priv::Allocator<phmap::priv::Pair<const K, V>>,
|
||||
size_t N = 4>
|
||||
using parallel_node_hash_map_m = parallel_node_hash_map<K, V, Hash, Eq, Alloc, N, std::mutex>;
|
||||
|
||||
// ------------- forward declarations for btree containers ----------------------------------
|
||||
template <typename Key, typename Compare = phmap::Less<Key>,
|
||||
typename Alloc = phmap::Allocator<Key>>
|
||||
|
||||
Reference in New Issue
Block a user