Skip to content

Commit

Permalink
fixed locality scoped locking
Browse files Browse the repository at this point in the history
  • Loading branch information
@ct-clmsn
ct-clmsn committed Apr 17, 2024
1 parent 83bda5e commit 79cbb02
Show file tree
Hide file tree
Showing 4 changed files with 24 additions and 21 deletions.
1 change: 1 addition & 0 deletions libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ namespace hpx { namespace util {
std::uint8_t* end_addr;
unsigned int * rcv;
unsigned int * xmt;
hpx::mutex * mut;
};

struct HPX_CORE_EXPORT openshmem_environment
Expand Down
3 changes: 2 additions & 1 deletion libs/core/openshmem_base/src/openshmem_environment.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

#include <hpx/config.hpp>
#include <hpx/assert.hpp>
#include <hpx/thread.hpp>
#include <hpx/modules/logging.hpp>
#include <hpx/modules/openshmem_base.hpp>
#include <hpx/modules/runtime_configuration.hpp>
Expand Down Expand Up @@ -204,6 +203,8 @@ namespace hpx::util {
//
segments[i].xmt = reinterpret_cast<unsigned int*>(hpx::util::openshmem_environment::shmem_buffer +
beg_signal + page_count + i);

segments[i].mut = &(openshmem_environment::segment_mutex[i]);
}

shmem_barrier_all();
Expand Down
22 changes: 11 additions & 11 deletions libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ namespace hpx::parcelset::policies::openshmem {
{
buffer_.chunks_.resize(num_zero_copy_chunks);
{
hpx::util::openshmem_environment::scoped_lock l;
std::lock_guard<hpx::mutex> l(*(hpx::util::openshmem_environment::segments[idx].mut));

hpx::util::openshmem_environment::wait_until(
1, hpx::util::openshmem_environment::segments[idx].rcv);
Expand Down Expand Up @@ -172,8 +172,9 @@ namespace hpx::parcelset::policies::openshmem {
const auto nthreads_ = hpx::util::openshmem_environment::nthreads_;
const auto idx = (self_*nthreads_)+sending_thd_id_;

hpx::util::openshmem_environment::scoped_lock l;
hpx::util::openshmem_environment::wait_until(
std::lock_guard<hpx::mutex> l(*(hpx::util::openshmem_environment::segments[idx].mut));

hpx::util::openshmem_environment::wait_until(
1, hpx::util::openshmem_environment::segments[idx].rcv);
(*(hpx::util::openshmem_environment::segments[idx].rcv)) = 0;

Expand Down Expand Up @@ -211,7 +212,7 @@ namespace hpx::parcelset::policies::openshmem {
const auto nthreads_ = hpx::util::openshmem_environment::nthreads_;
const auto idx = (self_*nthreads_)+sending_thd_id_;

hpx::util::openshmem_environment::scoped_lock l;
std::lock_guard<hpx::mutex> l(*(hpx::util::openshmem_environment::segments[idx].mut));

hpx::util::openshmem_environment::wait_until(
1, hpx::util::openshmem_environment::segments[idx].rcv);
Expand Down Expand Up @@ -249,7 +250,7 @@ namespace hpx::parcelset::policies::openshmem {
const auto nthreads_ = hpx::util::openshmem_environment::nthreads_;
const auto idx = (self_*nthreads_)+sending_thd_id_;

hpx::util::openshmem_environment::scoped_lock l;
std::lock_guard<hpx::mutex> l(*(hpx::util::openshmem_environment::segments[idx].mut));

hpx::util::openshmem_environment::wait_until(
1, hpx::util::openshmem_environment::segments[idx].rcv);
Expand Down Expand Up @@ -277,13 +278,12 @@ namespace hpx::parcelset::policies::openshmem {

bool request_done() noexcept
{
hpx::util::openshmem_environment::scoped_try_lock l;
if (!l.locked)
{
return false;
}
const auto self_ = hpx::util::openshmem_environment::rank();
const auto nthreads_ = hpx::util::openshmem_environment::nthreads_;
const auto idx = (self_*nthreads_)+sending_thd_id_;

return request_ptr_;
const bool l = hpx::util::openshmem_environment::segments[idx].mut->try_lock();
return l;
}

#if defined(HPX_HAVE_PARCELPORT_COUNTERS)
Expand Down
19 changes: 10 additions & 9 deletions libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ namespace hpx::parcelset::policies::openshmem {
const auto nthreads_ = hpx::util::openshmem_environment::nthreads_;
const auto idx = (dst_*nthreads_)+thd_id_;

hpx::util::openshmem_environment::scoped_lock l;
std::lock_guard<hpx::mutex> l(*(hpx::util::openshmem_environment::segments[idx].mut));

// put from this localities openshmem shared memory segment
// into the remote locality (dst_)'s shared memory segment
Expand Down Expand Up @@ -195,7 +195,7 @@ namespace hpx::parcelset::policies::openshmem {
const auto nthreads_ = hpx::util::openshmem_environment::nthreads_;
const auto idx = (dst_*nthreads_)+thd_id_;

hpx::util::openshmem_environment::scoped_lock l;
std::lock_guard<hpx::mutex> l(*(hpx::util::openshmem_environment::segments[idx].mut));

hpx::util::openshmem_environment::put_signal(
reinterpret_cast<std::uint8_t*>(chunks.data()), dst_,
Expand Down Expand Up @@ -225,7 +225,8 @@ namespace hpx::parcelset::policies::openshmem {
const auto nthreads_ = hpx::util::openshmem_environment::nthreads_;
const auto idx = (dst_*nthreads_)+thd_id_;

hpx::util::openshmem_environment::scoped_lock l;
std::lock_guard<hpx::mutex> l(*(hpx::util::openshmem_environment::segments[idx].mut));

hpx::util::openshmem_environment::put_signal(
reinterpret_cast<std::uint8_t*>(buffer_.data_.data()), dst_,
static_cast<std::uint8_t*>(hpx::util::openshmem_environment::segments[idx].beg_addr),
Expand Down Expand Up @@ -256,7 +257,7 @@ namespace hpx::parcelset::policies::openshmem {
return false;
}

hpx::util::openshmem_environment::scoped_lock l;
std::lock_guard<hpx::mutex> l(*(hpx::util::openshmem_environment::segments[idx].mut));

hpx::util::openshmem_environment::put_signal(
reinterpret_cast<const std::uint8_t*>(c.data_.cpos_),
Expand Down Expand Up @@ -304,11 +305,11 @@ namespace hpx::parcelset::policies::openshmem {

bool request_done()
{
hpx::util::openshmem_environment::scoped_try_lock l;
if (!l.locked)
{
return false;
}
const auto nthreads_ = hpx::util::openshmem_environment::nthreads_;
const auto idx = (dst_*nthreads_)+thd_id_;

const bool l = hpx::util::openshmem_environment::segments[idx].mut->try_lock();
return l;

return true;
}
Expand Down

0 comments on commit 79cbb02

Please sign in to comment.