diff --git a/CMakeLists.txt b/CMakeLists.txt index 28ce9c6..6df38d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -217,6 +217,11 @@ if (PHMAP_BUILD_EXAMPLES) target_compile_features(ex_llil4map PUBLIC cxx_std_20) target_link_libraries(ex_llil4map PRIVATE OpenMP::OpenMP_CXX) target_compile_options(ex_llil4map PRIVATE "${OpenMP_CXX_FLAGS}") + + add_executable(ex_llil examples/llil.cc phmap.natvis) + target_include_directories(ex_llil PRIVATE ${Boost_INCLUDE_DIRS}) + target_compile_features(ex_llil PUBLIC cxx_std_20) + file(COPY examples/llil_utils DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") endif() diff --git a/examples/llil.cc b/examples/llil.cc new file mode 100644 index 0000000..0d2e144 --- /dev/null +++ b/examples/llil.cc @@ -0,0 +1,366 @@ +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// llil4map.cc (new chunking variant) +// https://www.perlmonks.com/?node_id=11149643 +// A phmap::parallel_flat_hash_map demonstration. +// By Mario Roy, March 31, 2024 +// Based on llil3m.cpp https://perlmonks.com/?node_id=11149482 +// Original challenge https://perlmonks.com/?node_id=11147822 +// and summary https://perlmonks.com/?node_id=11150293 +// Other demonstrations https://perlmonks.com/?node_id=11149907 +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// OpenMP Little Book - https://nanxiao.gitbooks.io/openmp-little-book +// +// Obtain the parallel hashmap library (required dependency): +// git clone --depth=1 https://github.com/greg7mdp/parallel-hashmap +// +// Compile on Linux (clang++ or g++): +// clang++ -o llil4map -std=c++20 -fopenmp -Wall -O3 llil4map.cc -I./parallel-hashmap +// +// On macOS, use g++-12 from https://brew.sh (installation: brew install gcc@12). +// The g++ command also works with mingw C++ compiler (https://sourceforge.net/projects/mingw-w64) +// that comes bundled with Strawberry Perl (C:\Strawberry\c\bin\g++.exe). +// +// Obtain gen-llil.pl and gen-long-llil.pl from https://perlmonks.com/?node_id=11148681 +// perl gen-llil.pl big1.txt 200 3 1 +// perl gen-llil.pl big2.txt 200 3 1 +// perl gen-llil.pl big3.txt 200 3 1 +// perl gen-long-llil.pl long1.txt 600 +// perl gen-long-llil.pl long2.txt 600 +// perl gen-long-llil.pl long3.txt 600 +// +// To make random input, obtain shuffle.pl from https://perlmonks.com/?node_id=11149800 +// perl shuffle.pl big1.txt >tmp && mv tmp big1.txt +// perl shuffle.pl big2.txt >tmp && mv tmp big2.txt +// perl shuffle.pl big3.txt >tmp && mv tmp big3.txt +// +// Example run: llil4map big1.txt big2.txt big3.txt >out.txt +// NUM_THREADS=3 llil4map ... +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +// Specify 0/1 to use boost's parallel sorting algorithm; faster than __gnu_parallel::sort. +// https://www.boost.org/doc/libs/1_81_0/libs/sort/doc/html/sort/parallel.html +// This requires the boost header files: e.g. devpkg-boost bundle on Clear Linux. +// Note: Another option is downloading and unpacking Boost locally. +// (no need to build it because the bits we use are header file only) +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#if 1 + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wunused-parameter" + #pragma clang diagnostic ignored "-Wshadow" + + #include + + #pragma clang diagnostic pop +#endif + +#include +#include + +static_assert(sizeof(size_t) == sizeof(int64_t), "size_t too small, need a 64-bit compiler"); + +// ------------------------------------------------------------------------------------------ +template +struct show_time { + using high_resolution_clock = std::chrono::high_resolution_clock; + using time_point = std::chrono::high_resolution_clock::time_point; + using milliseconds = std::chrono::milliseconds; + + show_time(std::string_view message, F&& f) { + auto start = high_resolution_clock::now(); + std::forward(f)(); + auto elasped = double(std::chrono::duration_cast(high_resolution_clock::now() - start).count()) / 1000; + std::cerr << message << std::setw(8) << elasped << " secs\n"; + } +}; + + + +// ------------------------------------------------------------------------------------------ +struct stats_t { + using high_resolution_clock = std::chrono::high_resolution_clock; + using time_point = std::chrono::high_resolution_clock::time_point; + using milliseconds = std::chrono::milliseconds; + + struct time_pairs { + double elasped() const { return double(std::chrono::duration_cast(_stop - _start).count()) / 1000; } + void start() { _start = high_resolution_clock::now(); } + void stop() { _stop = high_resolution_clock::now(); } + //template + //void show_time(std::string_view message, F&& f) {} + + time_point _start, _stop; + }; + + time_pairs get_props; + time_pairs map_to_vec; + time_pairs sort; + time_pairs write_stdout; + time_pairs total; +}; + +// --------------------------------------------------------------------------------------------- +// Stores a string + a count +// For strings up to 11 bytes, total space used is 16 bytes (no wasted space in set) +// For larger strings, uses 16 bytes + strlen(string) + 1 +// +// invariants +// if extra[3], str is a valid string pointer +// if !extra[3], the 12 bytes starting at (const char *)(&str) store a null_terminated string +// --------------------------------------------------------------------------------------------- +struct string_cnt { + using uint_t = uint32_t; + + char * str; + char extra[4]; + uint_t cnt; + + static constexpr size_t buffsz = sizeof(str) + sizeof(extra); + + string_cnt() : str{nullptr}, extra{0,0,0,0}, cnt{0} {} + + string_cnt(std::string_view s, uint_t c = 0) : str(nullptr), cnt(c) { set(s); } + + ~string_cnt() { free(); } + + string_cnt(const string_cnt& o) { + set(o.get()); + } + + string_cnt(string_cnt&& o) noexcept { + if (o.extra[3]) { + str = o.str; + o.str = nullptr; + extra[3] = 1; + } else { + std::strcpy((char *)(&str), o.get()); + extra[3] = 0; + } + cnt = o.cnt; + } + + string_cnt& operator=(const string_cnt& o) { + free(); + set(o.get()); + cnt = o.cnt; + return *this; + } + + string_cnt& operator=(string_cnt&& o) noexcept { + free(); + new (this) string_cnt(std::move(o)); + return *this; + } + + std::strong_ordering operator<=>(const string_cnt& o) const { return std::strcmp(get(), o.get()) <=> 0; } + bool operator==(const string_cnt& o) const { return std::strcmp(get(), o.get()) == 0; } + + std::size_t hash() const { + auto s = get(); + std::string_view sv {s}; + return std::hash()(sv); + } + + const char *get() const { return extra[3] ? str : (const char *)(&str); } + +private: + void free() { if (extra[3]) { delete [] str; str = nullptr; } } + + void set(std::string_view s) { + static_assert(buffsz == 12); + static_assert(offsetof(string_cnt, cnt) == (intptr_t)buffsz); + static_assert(sizeof(string_cnt) == 16); + + assert(!extra[3] || !str); + if (s.empty()) + std::memset(&str, 0, buffsz); + else { + auto len = s.size(); + if (len >= buffsz) { + str = new char[len+1]; + std::memcpy(str, s.data(), len); + str[len] = 0; + extra[3] = 1; + } else { + std::memcpy((char *)(&str), s.data(), len); + ((char *)&str)[len] = 0; + extra[3] = 0; + } + } + } + + void set(const char *s) { + set(std::string_view{s}); + } +}; + +namespace std { + template<> struct hash { + std::size_t operator()(const string_cnt& v) const noexcept { return v.hash(); }; + }; +} + +namespace fs = std::filesystem; +namespace bip = boost::interprocess; +namespace lf = boost::lockfree; + +// ------------------------------------------------------------------------------------------ +template +class llil_t { +public: + using uint_t = string_cnt::uint_t; + static_assert(std::bit_ceil(num_wthreads) == num_wthreads); + + using string_cnt_set_t = phmap::parallel_flat_hash_set< + string_cnt, + phmap::priv::hash_default_hash, + phmap::priv::hash_default_eq, + phmap::priv::Allocator, + std::countr_zero(std::bit_ceil(num_wthreads)) + 1 + >; + + using string_cnt_vector_t = std::vector; + using string_cnt_vector_t2 = std::array; + + struct write_thread { + lf::queue queue; + std::thread thread; + }; + + string_cnt_set_t set; + std::atomic num_lines = 0; + lf::queue file_queue; + + void get_properties(const char* fname) { + auto mapping = bip::file_mapping(fname, bip::read_only); + auto rgn = bip::mapped_region(mapping, bip::read_only); + char* first = (char *)rgn.get_address(); + char* last = first + rgn.get_size(); + + size_t _num_lines = 0; + + std::array vecs; + constexpr size_t batch_size = 1024; + for (auto& v : vecs) + v.reserve(batch_size); + + while (first < last) { + char* beg_ptr{first}; + char* end_ptr{find_char(first, last, '\n')}; + char *found = find_char(beg_ptr, end_ptr, '\t'); + if (found == end_ptr) + continue; + + assert(*found == '\t'); + std::string_view word{beg_ptr, static_cast(found - beg_ptr)}; + uint_t count = fast_atoui(found + 1, end_ptr); + + size_t hashval = std::hash()(word); + auto& v = vecs[set.subidx(hashval)]; + v.emplace_back(word, count); + if (v.size() == batch_size) + v.resize(0); // actually should enqueue to other thread + + first = end_ptr + 1; + ++_num_lines; + } + + num_lines += _num_lines; + } + + template + void get_properties(char** fname, int nfiles) { + std::vector thr; + std::atomic done {false}; + + thr.reserve(num_producers); + + for (size_t i=0; iout.txt\n"; return 1; } + + llil_t<1> llil; + + show_time("get properties ", [&]() { llil.get_properties<6>(&argv[1], argc - 1); }); + + + llil.show_stats(); + + return 0; +} \ No newline at end of file