From 6105977a274a4fdab5e246d904174a055f31cae5 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Thu, 25 Jun 2020 16:05:29 +0200 Subject: [PATCH] DART-MPI: call into MPI every once in a while for local put/get --- dart-impl/mpi/src/dart_communication.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c index b4da40d73..a607c023a 100644 --- a/dart-impl/mpi/src/dart_communication.c +++ b/dart-impl/mpi/src/dart_communication.c @@ -30,6 +30,11 @@ #include #include +/* the number of consecutive memcpy for local put/get before calling into MPI */ +#define NUM_CONSECUTIVE_MEMCPY 16 + +/* number of performed local memcpy between calling into MPI */ +static _Thread_local int num_local_memcpy = 0; #define CHECK_UNITID_RANGE(_unitid, _team_data) \ do { \ @@ -220,13 +225,15 @@ dart__mpi__get_basic( { if (num_reqs) *num_reqs = 0; - if (team_data->unitid == team_unit_id.id) { + if (team_data->unitid == team_unit_id.id && + num_local_memcpy++ < NUM_CONSECUTIVE_MEMCPY) { // use direct memcpy if we are on the same unit memcpy(dest, seginfo->selfbaseptr + offset, nelem * dart__mpi__datatype_sizeof(dtype)); DART_LOG_DEBUG("dart_get: memcpy nelem:%zu " "source (coll.): offset:%lu -> dest: %p", nelem, offset, dest); + num_local_memcpy = 0; return DART_OK; } @@ -354,12 +361,14 @@ dart__mpi__put_basic( if (num_reqs) *num_reqs = 0; /* copy data directly if we are on the same unit */ - if (team_unit_id.id == team_data->unitid) { + if (team_unit_id.id == team_data->unitid && + num_local_memcpy++ < NUM_CONSECUTIVE_MEMCPY) { if (flush_required_ptr) *flush_required_ptr = false; memcpy(seginfo->selfbaseptr + offset, src, nelem * dart__mpi__datatype_sizeof(dtype)); DART_LOG_DEBUG("dart_put: memcpy nelem:%zu (from global allocation)" "offset: %"PRIu64"", nelem, offset); + num_local_memcpy = 0; return DART_OK; }